From ae206694e68bea074aca633ea0d32e9ed882a95f Mon Sep 17 00:00:00 2001
From: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Date: Thu, 11 May 2023 05:19:31 +0100
Subject: html builder: Append CRC32 checksum to asset URIs (#11415)
---
CHANGES | 3 +++
sphinx/builders/__init__.py | 7 +++++++
sphinx/builders/html/__init__.py | 41 +++++++++++++++++++++++++++++++--------
sphinx/builders/latex/__init__.py | 11 +++++++----
sphinx/builders/texinfo.py | 3 ++-
sphinx/ext/graphviz.py | 16 +++++++--------
tests/test_build_html.py | 35 ++++++++++++++++++++++-----------
tests/test_theming.py | 4 ++--
8 files changed, 86 insertions(+), 34 deletions(-)
diff --git a/CHANGES b/CHANGES
index cbc9d348c..7adb7cad5 100644
--- a/CHANGES
+++ b/CHANGES
@@ -22,6 +22,9 @@ Deprecated
Features added
--------------
+* #11415: Add a checksum to JavaScript and CSS asset URIs included within
+ generated HTML, using the CRC32 algorithm.
+
Bugs fixed
----------
diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py
index 852f25259..a5d5a1ef6 100644
--- a/sphinx/builders/__init__.py
+++ b/sphinx/builders/__init__.py
@@ -560,6 +560,9 @@ class Builder:
with progress_message(__('preparing documents')):
self.prepare_writing(docnames)
+ with progress_message(__('copying assets')):
+ self.copy_assets()
+
if self.parallel_ok:
# number of subprocesses is parallel-1 because the main process
# is busy loading doctrees and doing write_doc_serialized()
@@ -620,6 +623,10 @@ class Builder:
"""A place where you can add logic before :meth:`write_doc` is run"""
raise NotImplementedError
+ def copy_assets(self) -> None:
+ """Where assets (images, static files, etc) are copied before writing"""
+ pass
+
def write_doc(self, docname: str, doctree: nodes.document) -> None:
"""Where you actually write something to the filesystem."""
raise NotImplementedError
diff --git a/sphinx/builders/html/__init__.py b/sphinx/builders/html/__init__.py
index 8b8c426a4..64a5c1f60 100644
--- a/sphinx/builders/html/__init__.py
+++ b/sphinx/builders/html/__init__.py
@@ -8,6 +8,7 @@ import posixpath
import re
import sys
import warnings
+import zlib
from datetime import datetime
from os import path
from typing import IO, Any, Iterable, Iterator, List, Tuple, Type
@@ -649,6 +650,12 @@ class StandaloneHTMLBuilder(Builder):
'page_source_suffix': source_suffix,
}
+ def copy_assets(self) -> None:
+ self.finish_tasks.add_task(self.copy_download_files)
+ self.finish_tasks.add_task(self.copy_static_files)
+ self.finish_tasks.add_task(self.copy_extra_files)
+ self.finish_tasks.join()
+
def write_doc(self, docname: str, doctree: nodes.document) -> None:
destination = StringOutput(encoding='utf-8')
doctree.settings = self.docsettings
@@ -678,9 +685,6 @@ class StandaloneHTMLBuilder(Builder):
self.finish_tasks.add_task(self.gen_pages_from_extensions)
self.finish_tasks.add_task(self.gen_additional_pages)
self.finish_tasks.add_task(self.copy_image_files)
- self.finish_tasks.add_task(self.copy_download_files)
- self.finish_tasks.add_task(self.copy_static_files)
- self.finish_tasks.add_task(self.copy_extra_files)
self.finish_tasks.add_task(self.write_buildinfo)
# dump the search index
@@ -1193,8 +1197,11 @@ def setup_css_tag_helper(app: Sphinx, pagename: str, templatename: str,
value = css.attributes[key]
if value is not None:
attrs.append(f'{key}="{html.escape(value, True)}"')
- attrs.append('href="%s"' % pathto(css.filename, resource=True))
- return '' % ' '.join(attrs)
+ uri = pathto(css.filename, resource=True)
+ if checksum := _file_checksum(app.outdir, css.filename):
+ uri += f'?v={checksum}'
+ attrs.append(f'href="{uri}"')
+ return f''
context['css_tag'] = css_tag
@@ -1217,14 +1224,17 @@ def setup_js_tag_helper(app: Sphinx, pagename: str, templatename: str,
if key == 'body':
body = value
elif key == 'data_url_root':
- attrs.append('data-url_root="%s"' % pathto('', resource=True))
+ attrs.append(f'data-url_root="{pathto("", resource=True)}"')
else:
attrs.append(f'{key}="{html.escape(value, True)}"')
if js.filename:
- attrs.append('src="%s"' % pathto(js.filename, resource=True))
+ uri = pathto(js.filename, resource=True)
+ if checksum := _file_checksum(app.outdir, js.filename):
+ uri += f'?v={checksum}'
+ attrs.append(f'src="{uri}"')
else:
# str value (old styled)
- attrs.append('src="%s"' % pathto(js, resource=True))
+ attrs.append(f'src="{pathto(js, resource=True)}"')
if attrs:
return f''
@@ -1234,6 +1244,21 @@ def setup_js_tag_helper(app: Sphinx, pagename: str, templatename: str,
context['js_tag'] = js_tag
+def _file_checksum(outdir: str, filename: str) -> str:
+ # Don't generate checksums for HTTP URIs
+ if '://' in filename:
+ return ''
+ try:
+ # Ensure universal newline mode is used to avoid checksum differences
+ with open(path.join(outdir, filename), encoding='utf-8') as f:
+ content = f.read().encode(encoding='utf-8')
+ except FileNotFoundError:
+ return ''
+ if not content:
+ return ''
+ return f'{zlib.crc32(content):08x}'
+
+
def setup_resource_paths(app: Sphinx, pagename: str, templatename: str,
context: dict, doctree: Node) -> None:
"""Set up relative resource paths."""
diff --git a/sphinx/builders/latex/__init__.py b/sphinx/builders/latex/__init__.py
index 335518f23..e3e4d5042 100644
--- a/sphinx/builders/latex/__init__.py
+++ b/sphinx/builders/latex/__init__.py
@@ -254,6 +254,12 @@ class LaTeXBuilder(Builder):
f.write('% Its contents depend on pygments_style configuration variable.\n\n')
f.write(highlighter.get_stylesheet())
+ def copy_assets(self) -> None:
+ self.copy_support_files()
+
+ if self.config.latex_additional_files:
+ self.copy_latex_additional_files()
+
def write(self, *ignored: Any) -> None:
docwriter = LaTeXWriter(self)
with warnings.catch_warnings():
@@ -267,6 +273,7 @@ class LaTeXBuilder(Builder):
self.init_document_data()
self.write_stylesheet()
+ self.copy_assets()
for entry in self.document_data:
docname, targetname, title, author, themename = entry[:5]
@@ -371,10 +378,6 @@ class LaTeXBuilder(Builder):
def finish(self) -> None:
self.copy_image_files()
self.write_message_catalog()
- self.copy_support_files()
-
- if self.config.latex_additional_files:
- self.copy_latex_additional_files()
@progress_message(__('copying TeX support files'))
def copy_support_files(self) -> None:
diff --git a/sphinx/builders/texinfo.py b/sphinx/builders/texinfo.py
index 078991d9d..0b642af4c 100644
--- a/sphinx/builders/texinfo.py
+++ b/sphinx/builders/texinfo.py
@@ -85,6 +85,7 @@ class TexinfoBuilder(Builder):
def write(self, *ignored: Any) -> None:
self.init_document_data()
+ self.copy_assets()
for entry in self.document_data:
docname, targetname, title, author = entry[:4]
targetname += '.texi'
@@ -168,7 +169,7 @@ class TexinfoBuilder(Builder):
pendingnode.replace_self(newnodes)
return largetree
- def finish(self) -> None:
+ def copy_assets(self) -> None:
self.copy_support_files()
def copy_image_files(self, targetname: str) -> None:
diff --git a/sphinx/ext/graphviz.py b/sphinx/ext/graphviz.py
index c0a99be08..37626e04f 100644
--- a/sphinx/ext/graphviz.py
+++ b/sphinx/ext/graphviz.py
@@ -8,7 +8,7 @@ import re
import subprocess
from os import path
from subprocess import CalledProcessError
-from typing import Any
+from typing import TYPE_CHECKING, Any
from docutils import nodes
from docutils.nodes import Node
@@ -20,7 +20,6 @@ from sphinx.errors import SphinxError
from sphinx.locale import _, __
from sphinx.util import logging, sha1
from sphinx.util.docutils import SphinxDirective, SphinxTranslator
-from sphinx.util.fileutil import copy_asset
from sphinx.util.i18n import search_image_for_language
from sphinx.util.nodes import set_source_info
from sphinx.util.osutil import ensuredir
@@ -31,6 +30,9 @@ from sphinx.writers.manpage import ManualPageTranslator
from sphinx.writers.texinfo import TexinfoTranslator
from sphinx.writers.text import TextTranslator
+if TYPE_CHECKING:
+ from sphinx.config import Config
+
logger = logging.getLogger(__name__)
@@ -391,11 +393,9 @@ def man_visit_graphviz(self: ManualPageTranslator, node: graphviz) -> None:
raise nodes.SkipNode
-def on_build_finished(app: Sphinx, exc: Exception) -> None:
- if exc is None and app.builder.format == 'html':
- src = path.join(sphinx.package_dir, 'templates', 'graphviz', 'graphviz.css')
- dst = path.join(app.outdir, '_static')
- copy_asset(src, dst)
+def on_config_inited(_app: Sphinx, config: Config) -> None:
+ css_path = path.join(sphinx.package_dir, 'templates', 'graphviz', 'graphviz.css')
+ config.html_static_path.append(css_path)
def setup(app: Sphinx) -> dict[str, Any]:
@@ -412,5 +412,5 @@ def setup(app: Sphinx) -> dict[str, Any]:
app.add_config_value('graphviz_dot_args', [], 'html')
app.add_config_value('graphviz_output_format', 'png', 'html')
app.add_css_file('graphviz.css')
- app.connect('build-finished', on_build_finished)
+ app.connect('config-inited', on_config_inited)
return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/tests/test_build_html.py b/tests/test_build_html.py
index 8591897fb..c6a362098 100644
--- a/tests/test_build_html.py
+++ b/tests/test_build_html.py
@@ -1186,19 +1186,32 @@ def test_assets_order(app):
content = (app.outdir / 'index.html').read_text(encoding='utf8')
# css_files
- expected = ['_static/early.css', '_static/pygments.css', '_static/alabaster.css',
- 'https://example.com/custom.css', '_static/normal.css', '_static/late.css',
- '_static/css/style.css', '_static/lazy.css']
- pattern = '.*'.join('href="%s"' % f for f in expected)
- assert re.search(pattern, content, re.S)
+ expected = [
+ '_static/early.css',
+ '_static/pygments.css?v=b3523f8e',
+ '_static/alabaster.css?v=039e1c02',
+ 'https://example.com/custom.css',
+ '_static/normal.css',
+ '_static/late.css',
+ '_static/css/style.css',
+ '_static/lazy.css',
+ ]
+ pattern = '.*'.join(f'href="{re.escape(f)}"' for f in expected)
+ assert re.search(pattern, content, re.DOTALL), content
# js_files
- expected = ['_static/early.js',
- '_static/doctools.js', '_static/sphinx_highlight.js',
- 'https://example.com/script.js', '_static/normal.js',
- '_static/late.js', '_static/js/custom.js', '_static/lazy.js']
- pattern = '.*'.join('src="%s"' % f for f in expected)
- assert re.search(pattern, content, re.S)
+ expected = [
+ '_static/early.js',
+ '_static/doctools.js?v=888ff710',
+ '_static/sphinx_highlight.js?v=4825356b',
+ 'https://example.com/script.js',
+ '_static/normal.js',
+ '_static/late.js',
+ '_static/js/custom.js',
+ '_static/lazy.js',
+ ]
+ pattern = '.*'.join(f'src="{re.escape(f)}"' for f in expected)
+ assert re.search(pattern, content, re.DOTALL), content
@pytest.mark.sphinx('html', testroot='html_assets')
diff --git a/tests/test_theming.py b/tests/test_theming.py
index 579289988..b7ccda95c 100644
--- a/tests/test_theming.py
+++ b/tests/test_theming.py
@@ -99,10 +99,10 @@ def test_dark_style(app, status, warning):
assert (app.outdir / '_static' / 'pygments_dark.css').exists()
result = (app.outdir / 'index.html').read_text(encoding='utf8')
- assert '' in result
+ assert '' in result
assert ('') in result
+ 'href="_static/pygments_dark.css?v=e15ddae3" />') in result
@pytest.mark.sphinx(testroot='theming')
--
cgit v1.2.1