Builds canonical in book url from a pages metadata.
975524
def get_canonical_url(metadata, request):
    """"""
    slug_title = u'/{}'.format('-'.join(metadata['title'].split()))
    settings = get_current_registry().settings

    canon_host = settings.get('canonical-hostname',
                              re.sub('archive.', '', request.host))
    if metadata['canonical'] is None:
        canon_url = request.route_url(
                'content',
                ident_hash=metadata['id'],
                ignore=slug_title)
    else:
        canon_url = request.route_url(
                'content',
                ident_hash=metadata['canonical'],
                separator=':',
                page_ident_hash=metadata['id'],
                ignore=slug_title)

    return re.sub(request.host, canon_host, canon_url)
1177124
def get_book_links(links):
    
    book_links = []

    for link in links:
        data = DOWNER.download(link + "1")
        dom = dhtmlparser.parseString(data)

        book_links.extend(_parse_book_links(dom))

        max_page = _get_max_page(dom)
        if max_page == 1:
            continue

        for i in range(max_page - 1):
            data = DOWNER.download(link + str(i + 2))

            book_links.extend(
                _parse_book_links(
                    dhtmlparser.parseString(data)
                )
            )

    return book_links
1061343
function canonical() {
  var tags = document.getElementsByTagName('link');
  // eslint-disable-next-line no-cond-assign
  for (var i = 0, tag; tag = tags[i]; i++) {
    if (tag.getAttribute('rel') === 'canonical') {
      return tag.getAttribute('href');
    }
  }
}
1233741
def _read_page(file_obj, page_header, column_metadata):
    """"""
    bytes_from_file = file_obj.read(page_header.compressed_page_size)
    codec = column_metadata.codec
    if codec is not None and codec != parquet_thrift.CompressionCodec.UNCOMPRESSED:
        if column_metadata.codec == parquet_thrift.CompressionCodec.SNAPPY:
            raw_bytes = snappy.decompress(bytes_from_file)
        elif column_metadata.codec == parquet_thrift.CompressionCodec.GZIP:
            io_obj = io.BytesIO(bytes_from_file)
            with gzip.GzipFile(fileobj=io_obj, mode='rb') as file_data:
                raw_bytes = file_data.read()
        else:
            raise ParquetFormatException(
                "Unsupported Codec: {0}".format(codec))
    else:
        raw_bytes = bytes_from_file

    if logger.isEnabledFor(logging.DEBUG):
        logger.debug(
            "Read page with compression type %s. Bytes %d -> %d",
            _get_name(parquet_thrift.CompressionCodec, codec),
            page_header.compressed_page_size,
            page_header.uncompressed_page_size)
    assert len(raw_bytes) == page_header.uncompressed_page_size, \
        "found {0} raw bytes (expected {1})".format(
            len(raw_bytes),
            page_header.uncompressed_page_size)
    return raw_bytes
817376
def adapt_single_html(html):
    
    html_root = etree.fromstring(html)

    metadata = parse_metadata(html_root.xpath('//*[@data-type="metadata"]')[0])
    id_ = metadata['cnx-archive-uri'] or 'book'

    binder = Binder(id_, metadata=metadata)
    nav_tree = parse_navigation_html_to_tree(html_root, id_)

    body = html_root.xpath('//xhtml:body', namespaces=HTML_DOCUMENT_NAMESPACES)
    _adapt_single_html_tree(binder, body[0], nav_tree, top_metadata=metadata)

    return binder
1349618
def page(wiki, commit)
      blob         = self.blob(wiki.repo)
      page         = wiki.page_class.new(wiki).populate(blob, self.dir)
      page.version = commit
      page
    end
19283
def build_pubnote(title, volume, page_start=None, page_end=None, artid=None):
    """"""
    if title and volume and artid and artid != page_start:
        pubnote_format = u'{title},{volume},{artid}'
    elif title and volume and page_start and page_end:
        pubnote_format = u'{title},{volume},{page_start}-{page_end}'
    elif title and volume and page_start:
        pubnote_format = u'{title},{volume},{page_start}'
    elif title and volume:
        pubnote_format = u'{title},{volume}'
    else:
        return None

    return pubnote_format.format(
        title=title, volume=volume, page_start=page_start, page_end=page_end, artid=artid
    )
1284401
public static String canonicalizeUrl(String url, String refer) {
        URL base;
        try {
            try {
                base = new URL(refer);
            } catch (MalformedURLException e) {
                // the base is unsuitable, but the attribute may be abs on its own, so try that
                URL abs = new URL(refer);
                return abs.toExternalForm();
            }
            // workaround: java resolves '//path/file + ?foo' to '//path/?foo', not '//path/file?foo' as desired
            if (url.startsWith("?"))
                url = base.getPath() + url;
            URL abs = new URL(base, url);
            return abs.toExternalForm();
        } catch (MalformedURLException e) {
            return "";
        }
    }
172236
private static Page findPage(
		ServletContext servletContext,
		HttpServletRequest req,
		HttpServletResponse resp,
		SemanticCMS semanticCMS
	) throws ServletException, IOException {
		// Path extra info not allowed
		if(req.getPathInfo() != null) {
			return null;
		}
		// Query string not allowed
		if(req.getQueryString() != null) {
			return null;
		}
		String basePath;
		{
			String servletPath = req.getServletPath();
			// Must end in expected extension
			if(!servletPath.endsWith(RssUtils.EXTENSION)) {
				return null;
			}
			basePath = servletPath.substring(0, servletPath.length() - RssUtils.EXTENSION.length());
		}
		// Try to find the page, jspx, then jsp, then direct URL without extension
		String pagePath = null;
		for (String extension : RssUtils.getResourceExtensions()) {
			pagePath = basePath + extension;
			try {
				if(
					!RssUtils.isProtectedExtension(pagePath)
					&& ServletContextCache.getResource(servletContext, pagePath) != null
				) {
					break;
				}
			} catch(MalformedURLException e) {
				// Assume does not exist
			}
		}
		assert pagePath != null : "The last extension should be the default if none matched";
		if(RssUtils.isProtectedExtension(pagePath)) {
			return null;
		}
		// Find book and path
		PageRef pageRef;
		{
			Book book = semanticCMS.getPublishedBook(pagePath);
			if(book == null || !book.isAccessible()) {
				return null;
			}
			BookRef bookRef = book.getBookRef();
			try {
				pageRef = new PageRef(
					bookRef,
					Path.valueOf(pagePath.substring(bookRef.getPrefix().length()))
				);
			} catch(ValidationException e) {
				return null;
			}
		}
		// Capture the page
		return CapturePage.capturePage(
			servletContext,
			req,
			resp,
			pageRef,
			CaptureLevel.META
		);
	}
616248
function pageDefaults() {
  return {
    path: canonicalPath(),
    referrer: document.referrer,
    search: location.search,
    title: document.title,
    url: canonicalUrl(location.search)
  };
}
293113
public function setupSource( $respect_book_license = true ) {
		if ( ! empty( $this->sourceBookId ) ) {
			// Local book
			switch_to_blog( $this->sourceBookId );
		} elseif ( ! $this->isCompatible( $this->sourceBookUrl ) ) {
			// Remote is not compatible, bail.
			$_SESSION['pb_errors'][] = __( 'You can only clone from a book hosted by Pressbooks 4.1 or later. Please ensure that your source book meets these requirements.', 'pressbooks' );
			return false;
		}

		// Set up $this->sourceBookMetadata
		$this->sourceBookMetadata = $this->getBookMetadata( $this->sourceBookUrl );
		if ( empty( $this->sourceBookMetadata ) ) {
			$_SESSION['pb_errors'][] = sprintf( __( 'Could not retrieve metadata from %s.', 'pressbooks' ), sprintf( '<em>%s</em>', $this->sourceBookUrl ) );
			$this->maybeRestoreCurrentBlog();
			return false;
		}

		if ( $respect_book_license ) {
			// Verify license or network administrator override
			if ( ! $this->isSourceCloneable( $this->sourceBookMetadata['license'] ) ) {
				$_SESSION['pb_errors'][] = sprintf( __( '%s is not licensed for cloning.', 'pressbooks' ), sprintf( '<em>%s</em>', $this->sourceBookMetadata['name'] ) );
				$this->maybeRestoreCurrentBlog();
				return false;
			}
		}

		// Set up $this->sourceBookStructure
		$this->sourceBookStructure = $this->getBookStructure( $this->sourceBookUrl );
		if ( empty( $this->sourceBookStructure ) ) {
			$_SESSION['pb_errors'][] = sprintf( __( 'Could not retrieve contents and structure from %s.', 'pressbooks' ), sprintf( '<em>%s</em>', $this->sourceBookMetadata['name'] ) );
			$this->maybeRestoreCurrentBlog();
			return false;
		}

		// Set up $this->sourceBookTerms
		$this->sourceBookTerms = $this->getBookTerms( $this->sourceBookUrl );
		if ( empty( $this->sourceBookTerms ) ) {
			$_SESSION['pb_errors'][] = sprintf( __( 'Could not retrieve taxonomies from %s.', 'pressbooks' ), sprintf( '<em>%s</em>', $this->sourceBookMetadata['name'] ) );
			$this->maybeRestoreCurrentBlog();
			return false;
		}

		// Media
		$this->knownMedia = $this->buildListOfKnownMedia( $this->sourceBookUrl );
		if ( $this->knownMedia === false ) {
			$_SESSION['pb_errors'][] = sprintf( __( 'Could not retrieve media from %s.', 'pressbooks' ), sprintf( '<em>%s</em>', $this->sourceBookMetadata['name'] ) );
			$this->maybeRestoreCurrentBlog();
			return false;
		}
		// Sort by the length of sourceUrls for better, left to right, search and replace loops
		$known_media_sorted = $this->knownMedia;
		uasort(
			$known_media_sorted, function ( $a, $b ) {
				return strlen( $b->sourceUrl ) <=> strlen( $a->sourceUrl );
			}
		);
		$this->knownMedia = $known_media_sorted;

		// H5P
		$this->knownH5P = $this->buildListOfKnownH5P( $this->sourceBookUrl );
		if ( $this->knownH5P === false ) {
			// No H5P endpoint was found
			$this->knownH5P = [];
		} else {
			$this->sourceHasH5pApi = true;
		}

		// Set up $this->sourceBookGlossary
		$this->sourceBookGlossary = $this->getBookGlossary( $this->sourceBookUrl );

		$this->maybeRestoreCurrentBlog();
		return true;
	}
1008602
function canonicalPath() {
  var canon = canonical();
  if (!canon) return window.location.pathname;
  var parsed = url.parse(canon);
  return parsed.pathname;
}
1106186
def header_canonical(self, header_name):
        """"""
        # Translate as stated in the docs:
        # https://docs.djangoproject.com/en/1.6/ref/request-response/#django.http.HttpRequest.META
        header_name = header_name.lower()
        if header_name == 'content-type':
            return 'CONTENT-TYPE'
        elif header_name == 'content-length':
            return 'CONTENT-LENGTH'
        return 'HTTP_%s' % header_name.replace('-', '_').upper()
450377
def extract_canonical(bs4):
    

    link_rel = bs4.select('link[rel="canonical"]')

    if link_rel.__len__() > 0:

        if link_rel[0].has_attr('href'):

            return link_rel[0]['href']

    return None
150933
def _isbn_pairing(items):
    
    NameWrapper = namedtuple("NameWrapper", ["name", "obj"])
    metas = map(
        lambda x: NameWrapper(_just_name(x.filename), x),
        filter(lambda x: isinstance(x, MetadataFile), items)
    )
    ebooks = map(
        lambda x: NameWrapper(_just_name(x.filename), x),
        filter(lambda x: isinstance(x, EbookFile), items)
    )

    # simple pairing mechanism, which shouldn't be O^2 complex, but something
    # slightly better
    metas = sorted(metas, key=lambda x: x.name)
    ebooks = sorted(ebooks, key=lambda x: x.name, reverse=True)
    while metas:
        meta = metas.pop()

        if not isbn_validator.is_valid_isbn(meta.name):
            continue

        if not ebooks:
            break

        ebook_index = _index(ebooks, meta.name, key=lambda x: x.name)

        if ebook_index >= 0:
            logger.debug(
                "Pairing '%s' and '%s'." % (
                    meta.obj.filename,
                    ebooks[ebook_index].obj.filename
                )
            )
            items.append(
                DataPair(
                    metadata_file=meta.obj,
                    ebook_file=ebooks[ebook_index].obj
                )
            )
            items.remove(meta.obj)
            items.remove(ebooks[ebook_index].obj)
            ebooks = ebooks[ebook_index+1:]

    return items
1303671
function initBook(root, opts) {
    var book = new Book(root, opts);
    var extensionToUse = '.md';

    var chaptersPaths = function(chapters) {
        return _.reduce(chapters || [], function(accu, chapter) {
            var o = {
                title: chapter.title
            };
            if (chapter.path) o.path = chapter.path;

            return accu.concat(
                [o].concat(chaptersPaths(chapter.articles))
            );
        }, []);
    };

    book.log.info.ln('init book at', root);
    return fs.mkdirp(root)
    .then(function() {
        book.log.info.ln('detect structure from SUMMARY (if it exists)');
        return book.parseSummary();
    })
    .fail(function() {
        return Q();
    })
    .then(function() {
        var summary = book.summaryFile || 'SUMMARY.md';
        var chapters = book.summary.chapters || [];
        extensionToUse = path.extname(summary);

        if (chapters.length === 0) {
            chapters = [
                {
                    title: 'Summary',
                    path: 'SUMMARY'+extensionToUse
                },
                {
                    title: 'Introduction',
                    path: 'README'+extensionToUse
                }
            ];
        }

        return Q(chaptersPaths(chapters));
    })
    .then(function(chapters) {
        // Create files that don't exist
        return Q.all(_.map(chapters, function(chapter) {
            if (!chapter.path) return Q();
            var absolutePath = path.resolve(book.root, chapter.path);

            return fs.exists(absolutePath)
            .then(function(exists) {
                if(exists) {
                    book.log.info.ln('found', chapter.path);
                    return;
                } else {
                    book.log.info.ln('create', chapter.path);
                }

                return fs.mkdirp(path.dirname(absolutePath))
                .then(function() {
                    return fs.writeFile(absolutePath, '# '+chapter.title+'\n');
                });
            });
        }));
    })
    .then(function() {
        book.log.info.ln('initialization is finished');
    });
}
1281859
protected function generateCanonicalUrl()
    {
        $extCompat7 = ExtensionManagementUtility::isLoaded('compatibility7');
        //User has specified a canonical URL in the page properties
        if (!empty($this->pageRecord['tx_metaseo_canonicalurl'])) {
            return $this->generateLink($this->pageRecord['tx_metaseo_canonicalurl']);
        }

        //Fallback to global settings to generate Url
        if (!empty($this->tsSetupSeo['canonicalUrl'])) {
            list($clUrl, $clLinkConf, $clDisableMpMode) = $this->detectCanonicalPage(
                $this->tsSetupSeo['canonicalUrl.']
            );
            if (!empty($clUrl) && isset($clLinkConf) && isset($clDisableMpMode)) {
                $url = $this->generateLink($clUrl, $clLinkConf, $clDisableMpMode);
                return $this->setFallbackProtocol(
                    $extCompat7 ? $this->pageRecord['url_scheme'] : null, //page properties protocol selection
                    $this->tsSetupSeo['canonicalUrl.']['fallbackProtocol'],
                    $url
                );
            }
        }

        return null;
    }