diff options
88 files changed, 2561 insertions, 778 deletions
@@ -2,6 +2,8 @@ build compressor/tests/static/CACHE compressor/tests/static/custom compressor/tests/static/js/066cd253eada.js +compressor/tests/static/test.txt* + dist MANIFEST *.pyc @@ -10,3 +12,4 @@ MANIFEST docs/_build/ .sass-cache .coverage +.tox diff --git a/.travis.yml b/.travis.yml index 319ed2b..81020b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,22 @@ language: python -python: - - "2.6" - - "2.7" before_install: - - export PIP_USE_MIRRORS=true - - export PIP_INDEX_URL=https://simple.crate.io/ - sudo apt-get update - sudo apt-get install csstidy libxml2-dev libxslt-dev install: - - pip install -e . - - pip install -r requirements/tests.txt Django==$DJANGO + - pip install tox coveralls script: - - make test + - tox env: - - DJANGO=1.3.7 - - DJANGO=1.4.5 - - DJANGO=1.5 -branches: - only: - - develop + - TOXENV=py33-1.6.X + - TOXENV=py32-1.6.X + - TOXENV=py27-1.6.X + - TOXENV=py26-1.6.X + - TOXENV=py33-1.5.X + - TOXENV=py32-1.5.X + - TOXENV=py27-1.5.X + - TOXENV=py26-1.5.X + - TOXENV=py27-1.4.X + - TOXENV=py26-1.4.X notifications: irc: "irc.freenode.org#django-compressor" +after_success: coveralls @@ -55,9 +55,11 @@ Jonathan Lukens Julian Scheid Julien Phalip Justin Lilly +Lucas Tan Luis Nell Lukas Lehner -Lukasz Balcerzak +Łukasz Balcerzak +Łukasz Langa Maciek Szczesniak Maor Ben-Dayan Mark Lavin @@ -89,4 +91,4 @@ Ulrich Petri Ulysses V Vladislav Poluhin wesleyb -Wilson Júnior
\ No newline at end of file +Wilson Júnior @@ -1,6 +1,6 @@ django_compressor ----------------- -Copyright (c) 2009-2013 Django Compressor authors (see AUTHORS file) +Copyright (c) 2009-2014 Django Compressor authors (see AUTHORS file) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in index c7d4c74..a470974 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,10 @@ include AUTHORS include README.rst include LICENSE +include Makefile +include tox.ini recursive-include docs * +recursive-include requirements * recursive-include compressor/templates/compressor *.html recursive-include compressor/tests/media *.js *.css *.png *.coffee recursive-include compressor/tests/test_templates *.html @@ -1,4 +1,11 @@ +testenv: + pip install -e . + pip install -r requirements/tests.txt + pip install Django + test: - flake8 compressor --ignore=E501,E128 + flake8 compressor --ignore=E501,E128,E701,E261,E301,E126,E127,E131 coverage run --branch --source=compressor `which django-admin.py` test --settings=compressor.test_settings compressor coverage report --omit=compressor/test*,compressor/filters/jsmin/rjsmin*,compressor/filters/cssmin/cssmin*,compressor/utils/stringformat* + +.PHONY: test @@ -1,12 +1,21 @@ Django Compressor ================= -.. image:: https://secure.travis-ci.org/jezdez/django_compressor.png?branch=develop +.. image:: https://coveralls.io/repos/django-compressor/django-compressor/badge.png?branch=develop + :target: https://coveralls.io/r/django-compressor/django-compressor?branch=develop + +.. image:: https://pypip.in/v/django_compressor/badge.png + :target: https://pypi.python.org/pypi/django_compressor + +.. image:: https://pypip.in/d/django_compressor/badge.png + :target: https://pypi.python.org/pypi/django_compressor + +.. image:: https://secure.travis-ci.org/django-compressor/django-compressor.png?branch=develop :alt: Build Status - :target: http://travis-ci.org/jezdez/django_compressor + :target: http://travis-ci.org/django-compressor/django-compressor Django Compressor combines and compresses linked and inline Javascript -or CSS in a Django templates into cacheable static files by using the +or CSS in a Django template into cacheable static files by using the ``compress`` template tag. HTML in between ``{% compress js/css %}`` and ``{% endcompress %}`` is @@ -19,10 +28,10 @@ compresses it using ``jsmin``. As the final result the template tag outputs a ``<script>`` or ``<link>`` tag pointing to the optimized file. These files are stored inside a folder -and given an unique name based on their content. Alternatively it can also +and given a unique name based on their content. Alternatively it can also return the resulting content to the original template directly. -Since the file name is dependend on the content these files can be given +Since the file name is dependent on the content these files can be given a far future expiration date without worrying about stale browser caches. The concatenation and compressing process can also be jump started outside @@ -39,9 +48,10 @@ html5lib_ based parser, as well as an abstract base class that makes it easy to write a custom parser. Django Compressor also comes with built-in support for `CSS Tidy`_, -`YUI CSS and JS`_ compressor, the Google's `Closure Compiler`_, a Python -port of Douglas Crockford's JSmin_, a Python port of the YUI CSS Compressor -cssmin_ and a filter to convert (some) images into `data URIs`_. +`YUI CSS and JS`_ compressor, `yUglify CSS and JS`_ compressor, the Google's +`Closure Compiler`_, a Python port of Douglas Crockford's JSmin_, a Python port +of the YUI CSS Compressor cssmin_ and a filter to convert (some) images into +`data URIs`_. If your setup requires a different compressor or other post-processing tool it will be fairly easy to implement a custom filter. Simply extend @@ -51,21 +61,21 @@ More documentation about the usage and settings of Django Compressor can be found on `django-compressor.readthedocs.org`_. The source code for Django Compressor can be found and contributed to on -`github.com/jezdez/django_compressor`_. There you can also file tickets. +`github.com/django-compressor/django-compressor`_. There you can also file tickets. -The `in-development version`_ of Django Compressor can be installed with -``pip install django_compressor==dev`` or ``easy_install django_compressor==dev``. +The in-development version of Django Compressor can be installed with +``pip install http://github.com/django-compressor/django-compressor/tarball/develop``. .. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ .. _html5lib: http://code.google.com/p/html5lib/ .. _CSS Tidy: http://csstidy.sourceforge.net/ .. _YUI CSS and JS: http://developer.yahoo.com/yui/compressor/ +.. _yUglify CSS and JS: https://github.com/yui/yuglify .. _Closure Compiler: http://code.google.com/closure/compiler/ .. _JSMin: http://www.crockford.com/javascript/jsmin.html .. _cssmin: https://github.com/zacharyvoase/cssmin .. _data URIs: http://en.wikipedia.org/wiki/Data_URI_scheme .. _django-compressor.readthedocs.org: http://django-compressor.readthedocs.org/en/latest/ -.. _github.com/jezdez/django_compressor: https://github.com/jezdez/django_compressor -.. _in-development version: http://github.com/jezdez/django_compressor/tarball/develop#egg=django_compressor-dev +.. _github.com/django-compressor/django-compressor: https://github.com/django-compressor/django-compressor diff --git a/compressor/__init__.py b/compressor/__init__.py index 6524f6c..fdbc16e 100644 --- a/compressor/__init__.py +++ b/compressor/__init__.py @@ -1,2 +1,2 @@ # following PEP 386 -__version__ = "1.3" +__version__ = "1.4a1" diff --git a/compressor/base.py b/compressor/base.py index 4e91d4d..de9c9ce 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -1,22 +1,24 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals import os import codecs -import urllib from django.core.files.base import ContentFile from django.template import Context from django.template.loader import render_to_string -from django.utils.encoding import smart_unicode from django.utils.importlib import import_module from django.utils.safestring import mark_safe -from compressor.cache import get_hexdigest, get_mtime +try: + from urllib.request import url2pathname +except ImportError: + from urllib import url2pathname +from compressor.cache import get_hexdigest, get_mtime from compressor.conf import settings from compressor.exceptions import (CompressorError, UncompressableFileError, FilterDoesNotExist) from compressor.filters import CompilerFilter -from compressor.storage import default_storage, compressor_file_storage +from compressor.storage import compressor_file_storage from compressor.signals import post_compress from compressor.utils import get_class, get_mod_func, staticfiles from compressor.utils.decorators import cached_property @@ -34,16 +36,21 @@ class Compressor(object): type = None def __init__(self, content=None, output_prefix=None, context=None, *args, **kwargs): - self.content = content or "" + self.content = content or "" # rendered contents of {% compress %} tag self.output_prefix = output_prefix or "compressed" self.output_dir = settings.COMPRESS_OUTPUT_DIR.strip('/') self.charset = settings.DEFAULT_CHARSET - self.storage = default_storage self.split_content = [] self.context = context or {} self.extra_context = {} self.all_mimetypes = dict(settings.COMPRESS_PRECOMPILERS) self.finders = staticfiles.finders + self._storage = None + + @cached_property + def storage(self): + from compressor.storage import default_storage + return default_storage def split_contents(self): """ @@ -65,6 +72,10 @@ class Compressor(object): return "compressor/%s_%s.html" % (self.type, mode) def get_basename(self, url): + """ + Takes full path to a static file (eg. "/static/css/style.css") and + returns path with storage's base url removed (eg. "css/style.css"). + """ try: base_url = self.storage.base_url except AttributeError: @@ -78,6 +89,17 @@ class Compressor(object): return basename.split("?", 1)[0] def get_filepath(self, content, basename=None): + """ + Returns file path for an output file based on contents. + + Returned path is relative to compressor storage's base url, for + example "CACHE/css/e41ba2cc6982.css". + + When `basename` argument is provided then file name (without extension) + will be used as a part of returned file name, for example: + + get_filepath(content, "my_file.css") -> 'CACHE/css/my_file.e41ba2cc6982.css' + """ parts = [] if basename: filename = os.path.split(basename)[1] @@ -86,6 +108,11 @@ class Compressor(object): return os.path.join(self.output_dir, self.output_prefix, '.'.join(parts)) def get_filename(self, basename): + """ + Returns full path to a file, for example: + + get_filename('css/one.css') -> '/full/path/to/static/css/one.css' + """ filename = None # first try finding the file in the root try: @@ -100,7 +127,7 @@ class Compressor(object): filename = compressor_file_storage.path(basename) # secondly try to find it with staticfiles (in debug mode) if not filename and self.finders: - filename = self.finders.find(urllib.url2pathname(basename)) + filename = self.finders.find(url2pathname(basename)) if filename: return filename # or just raise an exception as the last resort @@ -110,13 +137,16 @@ class Compressor(object): self.finders and " or with staticfiles." or ".")) def get_filecontent(self, filename, charset): - with codecs.open(filename, 'rb', charset) as fd: + """ + Reads file contents using given `charset` and returns it as text. + """ + with codecs.open(filename, 'r', charset) as fd: try: return fd.read() - except IOError, e: + except IOError as e: raise UncompressableFileError("IOError while processing " "'%s': %s" % (filename, e)) - except UnicodeDecodeError, e: + except UnicodeDecodeError as e: raise UncompressableFileError("UnicodeDecodeError while " "processing '%s' with " "charset %s: %s" % @@ -143,7 +173,7 @@ class Compressor(object): def hunks(self, forced=False): """ - The heart of content parsing, iterates of the + The heart of content parsing, iterates over the list of split contents and looks at its kind to decide what to do with it. Should yield a bunch of precompiled and/or rendered hunks. @@ -159,6 +189,7 @@ class Compressor(object): 'elem': elem, 'kind': kind, 'basename': basename, + 'charset': charset, } if kind == SOURCE_FILE: @@ -169,12 +200,11 @@ class Compressor(object): precompiled, value = self.precompile(value, **options) if enabled: - value = self.filter(value, **options) - yield smart_unicode(value, charset.lower()) + yield self.filter(value, **options) else: if precompiled: - value = self.handle_output(kind, value, forced=True, basename=basename) - yield smart_unicode(value, charset.lower()) + yield self.handle_output(kind, value, forced=True, + basename=basename) else: yield self.parser.elem_str(elem) @@ -195,7 +225,13 @@ class Compressor(object): content.append(hunk) return content - def precompile(self, content, kind=None, elem=None, filename=None, **kwargs): + def precompile(self, content, kind=None, elem=None, filename=None, + charset=None, **kwargs): + """ + Processes file using a pre compiler. + + This is the place where files like coffee script are processed. + """ if not kind: return False, content attrs = self.parser.elem_attribs(elem) @@ -212,18 +248,21 @@ class Compressor(object): try: mod = import_module(mod_name) except ImportError: - return True, CompilerFilter(content, filter_type=self.type, - command=filter_or_command, filename=filename).input( - **kwargs) + filter = CompilerFilter( + content, filter_type=self.type, filename=filename, + charset=charset, command=filter_or_command) + return True, filter.input(**kwargs) try: precompiler_class = getattr(mod, cls_name) except AttributeError: raise FilterDoesNotExist('Could not find "%s".' % filter_or_command) else: - return True, precompiler_class(content, attrs, - filter_type=self.type, filename=filename).input( - **kwargs) + filter = precompiler_class( + content, attrs, filter_type=self.type, charset=charset, + filename=filename) + return True, filter.input(**kwargs) + return False, content def filter(self, content, method, **kwargs): @@ -243,11 +282,10 @@ class Compressor(object): any custom modification. Calls other mode specific methods or simply returns the content directly. """ - content = self.filter_input(forced) - if not content: - return '' + output = '\n'.join(self.filter_input(forced)) - output = '\n'.join(c.encode(self.charset) for c in content) + if not output: + return '' if settings.COMPRESS_ENABLED or forced: filtered_output = self.filter_output(output) @@ -271,7 +309,7 @@ class Compressor(object): """ new_filepath = self.get_filepath(content, basename=basename) if not self.storage.exists(new_filepath) or forced: - self.storage.save(new_filepath, ContentFile(content)) + self.storage.save(new_filepath, ContentFile(content.encode(self.charset))) url = mark_safe(self.storage.url(new_filepath)) return self.render_output(mode, {"url": url}) diff --git a/compressor/cache.py b/compressor/cache.py index 1caeded..4847939 100644 --- a/compressor/cache.py +++ b/compressor/cache.py @@ -1,3 +1,4 @@ +import json import hashlib import os import socket @@ -5,8 +6,7 @@ import time from django.core.cache import get_cache from django.core.files.base import ContentFile -from django.utils import simplejson -from django.utils.encoding import smart_str +from django.utils.encoding import force_text, smart_bytes from django.utils.functional import SimpleLazyObject from django.utils.importlib import import_module @@ -18,18 +18,18 @@ _cachekey_func = None def get_hexdigest(plaintext, length=None): - digest = hashlib.md5(smart_str(plaintext)).hexdigest() + digest = hashlib.md5(smart_bytes(plaintext)).hexdigest() if length: return digest[:length] return digest def simple_cachekey(key): - return 'django_compressor.%s' % smart_str(key) + return 'django_compressor.%s' % force_text(key) def socket_cachekey(key): - return "django_compressor.%s.%s" % (socket.gethostname(), smart_str(key)) + return 'django_compressor.%s.%s' % (socket.gethostname(), force_text(key)) def get_cachekey(*args, **kwargs): @@ -39,7 +39,7 @@ def get_cachekey(*args, **kwargs): mod_name, func_name = get_mod_func( settings.COMPRESS_CACHE_KEY_FUNCTION) _cachekey_func = getattr(import_module(mod_name), func_name) - except (AttributeError, ImportError), e: + except (AttributeError, ImportError) as e: raise ImportError("Couldn't import cache key function %s: %s" % (settings.COMPRESS_CACHE_KEY_FUNCTION, e)) return _cachekey_func(*args, **kwargs) @@ -70,7 +70,8 @@ def get_offline_manifest(): if _offline_manifest is None: filename = get_offline_manifest_filename() if default_storage.exists(filename): - _offline_manifest = simplejson.load(default_storage.open(filename)) + with default_storage.open(filename) as fp: + _offline_manifest = json.loads(fp.read().decode('utf8')) else: _offline_manifest = {} return _offline_manifest @@ -83,8 +84,8 @@ def flush_offline_manifest(): def write_offline_manifest(manifest): filename = get_offline_manifest_filename() - default_storage.save(filename, - ContentFile(simplejson.dumps(manifest, indent=2))) + content = json.dumps(manifest, indent=2).encode('utf8') + default_storage.save(filename, ContentFile(content)) flush_offline_manifest() @@ -118,12 +119,10 @@ def get_hashed_content(filename, length=12): filename = os.path.realpath(filename) except OSError: return None - hash_file = open(filename) - try: - content = hash_file.read() - finally: - hash_file.close() - return get_hexdigest(content, length) + + # should we make sure that file is utf-8 encoded? + with open(filename, 'rb') as file: + return get_hexdigest(file.read(), length) def cache_get(key): diff --git a/compressor/conf.py b/compressor/conf.py index 5ba7bee..e9763d9 100644 --- a/compressor/conf.py +++ b/compressor/conf.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import os from django.conf import settings from django.core.exceptions import ImproperlyConfigured @@ -11,7 +12,7 @@ class CompressorConf(AppConf): # Allows changing verbosity from the settings. VERBOSE = False # GET variable that disables compressor e.g. "nocompress" - DEBUG_TOGGLE = 'None' + DEBUG_TOGGLE = None # the backend to use when parsing the JavaScript or Stylesheet files PARSER = 'compressor.parser.AutoSelectParser' OUTPUT_DIR = 'CACHE' @@ -41,6 +42,9 @@ class CompressorConf(AppConf): YUI_BINARY = 'java -jar yuicompressor.jar' YUI_CSS_ARGUMENTS = '' YUI_JS_ARGUMENTS = '' + YUGLIFY_BINARY = 'yuglify' + YUGLIFY_CSS_ARGUMENTS = '--terminal' + YUGLIFY_JS_ARGUMENTS = '--terminal' DATA_URI_MAX_SIZE = 1024 # the cache backend to use @@ -64,6 +68,13 @@ class CompressorConf(AppConf): OFFLINE_MANIFEST = 'manifest.json' # The Context to be used when TemplateFilter is used TEMPLATE_FILTER_CONTEXT = {} + # Function that returns the Jinja2 environment to use in offline compression. + def JINJA2_GET_ENVIRONMENT(): + try: + import jinja2 + return jinja2.Environment() + except ImportError: + return None class Meta: prefix = 'compress' @@ -73,7 +84,8 @@ class CompressorConf(AppConf): if value is None: value = settings.STATIC_ROOT if value is None: - raise ImproperlyConfigured("COMPRESS_ROOT setting must be set") + raise ImproperlyConfigured('COMPRESS_ROOT defaults to ' + + 'STATIC_ROOT, please define either') return os.path.normcase(os.path.abspath(value)) def configure_url(self, value): diff --git a/compressor/contrib/jinja2ext.py b/compressor/contrib/jinja2ext.py index baf76d5..7215d4d 100644 --- a/compressor/contrib/jinja2ext.py +++ b/compressor/contrib/jinja2ext.py @@ -10,7 +10,7 @@ class CompressorExtension(CompressorMixin, Extension): tags = set(['compress']) def parse(self, parser): - lineno = parser.stream.next().lineno + lineno = next(parser.stream).lineno kindarg = parser.parse_expression() # Allow kind to be defined as jinja2 name node if isinstance(kindarg, nodes.Name): @@ -28,14 +28,22 @@ class CompressorExtension(CompressorMixin, Extension): args.append(modearg) else: args.append(nodes.Const('file')) + body = parser.parse_statements(['name:endcompress'], drop_needle=True) - return nodes.CallBlock(self.call_method('_compress', args), [], [], + + # Skip the kind if used in the endblock, by using the kind in the + # endblock the templates are slightly more readable. + parser.stream.skip_if('name:' + kindarg.value) + return nodes.CallBlock(self.call_method('_compress_normal', args), [], [], body).set_lineno(lineno) - def _compress(self, kind, mode, caller): - # This extension assumes that we won't force compression - forced = False + def _compress_forced(self, kind, mode, caller): + return self._compress(kind, mode, caller, True) + + def _compress_normal(self, kind, mode, caller): + return self._compress(kind, mode, caller, False) + def _compress(self, kind, mode, caller, forced): mode = mode or OUTPUT_FILE original_content = caller() context = { diff --git a/compressor/contrib/sekizai.py b/compressor/contrib/sekizai.py index 7d5ac19..87966c5 100644 --- a/compressor/contrib/sekizai.py +++ b/compressor/contrib/sekizai.py @@ -2,7 +2,7 @@ source: https://gist.github.com/1311010 Get django-sekizai, django-compessor (and django-cms) playing nicely together re: https://github.com/ojii/django-sekizai/issues/4 - using: https://github.com/jezdez/django_compressor.git + using: https://github.com/django-compressor/django-compressor.git and: https://github.com/ojii/django-sekizai.git@0.6 or later """ from compressor.templatetags.compress import CompressorNode diff --git a/compressor/css.py b/compressor/css.py index ffd0069..e10697b 100644 --- a/compressor/css.py +++ b/compressor/css.py @@ -33,7 +33,7 @@ class CssCompressor(Compressor): if append_to_previous and settings.COMPRESS_ENABLED: self.media_nodes[-1][1].split_content.append(data) else: - node = CssCompressor(content=self.parser.elem_str(elem), + node = self.__class__(content=self.parser.elem_str(elem), context=self.context) node.split_content.append(data) self.media_nodes.append((media, node)) diff --git a/compressor/exceptions.py b/compressor/exceptions.py index 07d79a1..c2d7c60 100644 --- a/compressor/exceptions.py +++ b/compressor/exceptions.py @@ -38,3 +38,17 @@ class FilterDoesNotExist(Exception): Raised when a filter class cannot be found. """ pass + + +class TemplateDoesNotExist(Exception): + """ + This exception is raised when a template does not exist. + """ + pass + + +class TemplateSyntaxError(Exception): + """ + This exception is raised when a template syntax error is encountered. + """ + pass diff --git a/compressor/filters/base.py b/compressor/filters/base.py index 641cf6b..284afcb 100644 --- a/compressor/filters/base.py +++ b/compressor/filters/base.py @@ -1,27 +1,37 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals +import io import logging import subprocess from django.core.exceptions import ImproperlyConfigured from django.core.files.temp import NamedTemporaryFile from django.utils.importlib import import_module +from django.utils.encoding import smart_text +from django.utils import six from compressor.conf import settings from compressor.exceptions import FilterError from compressor.utils import get_mod_func -from compressor.utils.stringformat import FormattableString as fstr + logger = logging.getLogger("compressor.filters") class FilterBase(object): + """ + A base class for filters that does nothing. - def __init__(self, content, filter_type=None, filename=None, verbose=0): + Subclasses should implement `input` and/or `output` methods which must + return a string (unicode under python 2) or raise a NotImplementedError. + """ + def __init__(self, content, filter_type=None, filename=None, verbose=0, + charset=None): self.type = filter_type self.content = content self.verbose = verbose or settings.COMPRESS_VERBOSE self.logger = logger self.filename = filename + self.charset = charset def input(self, **kwargs): raise NotImplementedError @@ -31,6 +41,16 @@ class FilterBase(object): class CallbackOutputFilter(FilterBase): + """ + A filter which takes function path in `callback` attribute, imports it + and uses that function to filter output string:: + + class MyFilter(CallbackOutputFilter): + callback = 'path.to.my.callback' + + Callback should be a function which takes a string as first argument and + returns a string (unicode under python 2). + """ callback = None args = [] kwargs = {} @@ -39,12 +59,13 @@ class CallbackOutputFilter(FilterBase): def __init__(self, *args, **kwargs): super(CallbackOutputFilter, self).__init__(*args, **kwargs) if self.callback is None: - raise ImproperlyConfigured("The callback filter %s must define" - "a 'callback' attribute." % self) + raise ImproperlyConfigured( + "The callback filter %s must define a 'callback' attribute." % + self.__class__.__name__) try: mod_name, func_name = get_mod_func(self.callback) func = getattr(import_module(mod_name), func_name) - except ImportError, e: + except ImportError: if self.dependencies: if len(self.dependencies) == 1: warning = "dependency (%s) is" % self.dependencies[0] @@ -53,17 +74,19 @@ class CallbackOutputFilter(FilterBase): ", ".join([dep for dep in self.dependencies])) else: warning = "" - raise ImproperlyConfigured("The callback %s couldn't be imported. " - "Make sure the %s correctly installed." - % (self.callback, warning)) - except AttributeError, e: - raise ImproperlyConfigured("An error occured while importing the " + raise ImproperlyConfigured( + "The callback %s couldn't be imported. Make sure the %s " + "correctly installed." % (self.callback, warning)) + except AttributeError as e: + raise ImproperlyConfigured("An error occurred while importing the " "callback filter %s: %s" % (self, e)) else: self._callback_func = func def output(self, **kwargs): - return self._callback_func(self.content, *self.args, **self.kwargs) + ret = self._callback_func(self.content, *self.args, **self.kwargs) + assert isinstance(ret, six.text_type) + return ret class CompilerFilter(FilterBase): @@ -73,71 +96,93 @@ class CompilerFilter(FilterBase): """ command = None options = () + default_encoding = settings.FILE_CHARSET def __init__(self, content, command=None, *args, **kwargs): super(CompilerFilter, self).__init__(content, *args, **kwargs) self.cwd = None + if command: self.command = command if self.command is None: raise FilterError("Required attribute 'command' not given") + if isinstance(self.options, dict): + # turn dict into a tuple new_options = () - for item in kwargs.iteritems(): + for item in kwargs.items(): new_options += (item,) self.options = new_options - for item in kwargs.iteritems(): + + # append kwargs to self.options + for item in kwargs.items(): self.options += (item,) - self.stdout = subprocess.PIPE - self.stdin = subprocess.PIPE - self.stderr = subprocess.PIPE - self.infile, self.outfile = None, None + + self.stdout = self.stdin = self.stderr = subprocess.PIPE + self.infile = self.outfile = None def input(self, **kwargs): + encoding = self.default_encoding options = dict(self.options) - if self.infile is None: - if "{infile}" in self.command: - if self.filename is None: - self.infile = NamedTemporaryFile(mode="w") - self.infile.write(self.content.encode('utf8')) - self.infile.flush() - options["infile"] = self.infile.name - else: - self.infile = open(self.filename) - options["infile"] = self.filename - if "{outfile}" in self.command and not "outfile" in options: + if self.infile is None and "{infile}" in self.command: + # create temporary input file if needed + if self.filename is None: + self.infile = NamedTemporaryFile(mode='wb') + self.infile.write(self.content.encode(encoding)) + self.infile.flush() + options["infile"] = self.infile.name + else: + # we use source file directly, which may be encoded using + # something different than utf8. If that's the case file will + # be included with charset="something" html attribute and + # charset will be available as filter's charset attribute + encoding = self.charset # or self.default_encoding + self.infile = open(self.filename) + options["infile"] = self.filename + + if "{outfile}" in self.command and "outfile" not in options: + # create temporary output file if needed ext = self.type and ".%s" % self.type or "" self.outfile = NamedTemporaryFile(mode='r+', suffix=ext) options["outfile"] = self.outfile.name + try: - command = fstr(self.command).format(**options) - proc = subprocess.Popen(command, shell=True, cwd=self.cwd, - stdout=self.stdout, stdin=self.stdin, stderr=self.stderr) + command = self.command.format(**options) + proc = subprocess.Popen( + command, shell=True, cwd=self.cwd, stdout=self.stdout, + stdin=self.stdin, stderr=self.stderr) if self.infile is None: - filtered, err = proc.communicate(self.content.encode('utf8')) + # if infile is None then send content to process' stdin + filtered, err = proc.communicate( + self.content.encode(encoding)) else: filtered, err = proc.communicate() - except (IOError, OSError), e: + filtered, err = filtered.decode(encoding), err.decode(encoding) + except (IOError, OSError) as e: raise FilterError('Unable to apply %s (%r): %s' % (self.__class__.__name__, self.command, e)) else: if proc.wait() != 0: + # command failed, raise FilterError exception if not err: err = ('Unable to apply %s (%s)' % (self.__class__.__name__, self.command)) if filtered: err += '\n%s' % filtered raise FilterError(err) + if self.verbose: self.logger.debug(err) + outfile_path = options.get('outfile') if outfile_path: - self.outfile = open(outfile_path, 'r') + with io.open(outfile_path, 'r', encoding=encoding) as file: + filtered = file.read() finally: if self.infile is not None: self.infile.close() if self.outfile is not None: - filtered = self.outfile.read() self.outfile.close() - return filtered + + return smart_text(filtered) diff --git a/compressor/filters/cssmin/__init__.py b/compressor/filters/cssmin/__init__.py index a71f016..073303d 100644 --- a/compressor/filters/cssmin/__init__.py +++ b/compressor/filters/cssmin/__init__.py @@ -7,3 +7,7 @@ class CSSMinFilter(CallbackOutputFilter): the YUI CSS compression algorithm: http://pypi.python.org/pypi/cssmin/ """ callback = "compressor.filters.cssmin.cssmin.cssmin" + + +class rCSSMinFilter(CallbackOutputFilter): + callback = "compressor.filters.cssmin.rcssmin.cssmin" diff --git a/compressor/filters/cssmin/cssmin.py b/compressor/filters/cssmin/cssmin.py index 3dc0cc7..e8a02b0 100644 --- a/compressor/filters/cssmin/cssmin.py +++ b/compressor/filters/cssmin/cssmin.py @@ -28,14 +28,8 @@ # """`cssmin` - A Python port of the YUI CSS compressor.""" - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO # noqa import re - __version__ = '0.1.4' diff --git a/compressor/filters/cssmin/rcssmin.py b/compressor/filters/cssmin/rcssmin.py new file mode 100644 index 0000000..ff8e273 --- /dev/null +++ b/compressor/filters/cssmin/rcssmin.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python +# -*- coding: ascii -*- +# +# Copyright 2011, 2012 +# Andr\xe9 Malo or his licensors, as applicable +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r""" +============== + CSS Minifier +============== + +CSS Minifier. + +The minifier is based on the semantics of the `YUI compressor`_\, which itself +is based on `the rule list by Isaac Schlueter`_\. + +This module is a re-implementation aiming for speed instead of maximum +compression, so it can be used at runtime (rather than during a preprocessing +step). RCSSmin does syntactical compression only (removing spaces, comments +and possibly semicolons). It does not provide semantic compression (like +removing empty blocks, collapsing redundant properties etc). It does, however, +support various CSS hacks (by keeping them working as intended). + +Here's a feature list: + +- Strings are kept, except that escaped newlines are stripped +- Space/Comments before the very end or before various characters are + stripped: ``:{});=>+],!`` (The colon (``:``) is a special case, a single + space is kept if it's outside a ruleset.) +- Space/Comments at the very beginning or after various characters are + stripped: ``{}(=:>+[,!`` +- Optional space after unicode escapes is kept, resp. replaced by a simple + space +- whitespaces inside ``url()`` definitions are stripped +- Comments starting with an exclamation mark (``!``) can be kept optionally. +- All other comments and/or whitespace characters are replaced by a single + space. +- Multiple consecutive semicolons are reduced to one +- The last semicolon within a ruleset is stripped +- CSS Hacks supported: + + - IE7 hack (``>/**/``) + - Mac-IE5 hack (``/*\*/.../**/``) + - The boxmodelhack is supported naturally because it relies on valid CSS2 + strings + - Between ``:first-line`` and the following comma or curly brace a space is + inserted. (apparently it's needed for IE6) + - Same for ``:first-letter`` + +rcssmin.c is a reimplementation of rcssmin.py in C and improves runtime up to +factor 50 or so (depending on the input). + +Both python 2 (>= 2.4) and python 3 are supported. + +.. _YUI compressor: https://github.com/yui/yuicompressor/ + +.. _the rule list by Isaac Schlueter: https://github.com/isaacs/cssmin/tree/ +""" +__author__ = "Andr\xe9 Malo" +__author__ = getattr(__author__, 'decode', lambda x: __author__)('latin-1') +__docformat__ = "restructuredtext en" +__license__ = "Apache License, Version 2.0" +__version__ = '1.0.2' +__all__ = ['cssmin'] + +import re as _re + + +def _make_cssmin(python_only=False): + """ + Generate CSS minifier. + + :Parameters: + `python_only` : ``bool`` + Use only the python variant. If true, the c extension is not even + tried to be loaded. + + :Return: Minifier + :Rtype: ``callable`` + """ + # pylint: disable = W0612 + # ("unused" variables) + + # pylint: disable = R0911, R0912, R0914, R0915 + # (too many anything) + + if not python_only: + try: + import _rcssmin + except ImportError: + pass + else: + return _rcssmin.cssmin + + nl = r'(?:[\n\f]|\r\n?)' # pylint: disable = C0103 + spacechar = r'[\r\n\f\040\t]' + + unicoded = r'[0-9a-fA-F]{1,6}(?:[\040\n\t\f]|\r\n?)?' + escaped = r'[^\n\r\f0-9a-fA-F]' + escape = r'(?:\\(?:%(unicoded)s|%(escaped)s))' % locals() + + nmchar = r'[^\000-\054\056\057\072-\100\133-\136\140\173-\177]' + # nmstart = r'[^\000-\100\133-\136\140\173-\177]' + # ident = (r'(?:' + # r'-?(?:%(nmstart)s|%(escape)s)%(nmchar)s*(?:%(escape)s%(nmchar)s*)*' + # r')') % locals() + + comment = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)' + + # only for specific purposes. The bang is grouped: + _bang_comment = r'(?:/\*(!?)[^*]*\*+(?:[^/*][^*]*\*+)*/)' + + string1 = \ + r'(?:\047[^\047\\\r\n\f]*(?:\\[^\r\n\f][^\047\\\r\n\f]*)*\047)' + string2 = r'(?:"[^"\\\r\n\f]*(?:\\[^\r\n\f][^"\\\r\n\f]*)*")' + strings = r'(?:%s|%s)' % (string1, string2) + + nl_string1 = \ + r'(?:\047[^\047\\\r\n\f]*(?:\\(?:[^\r]|\r\n?)[^\047\\\r\n\f]*)*\047)' + nl_string2 = r'(?:"[^"\\\r\n\f]*(?:\\(?:[^\r]|\r\n?)[^"\\\r\n\f]*)*")' + nl_strings = r'(?:%s|%s)' % (nl_string1, nl_string2) + + uri_nl_string1 = r'(?:\047[^\047\\]*(?:\\(?:[^\r]|\r\n?)[^\047\\]*)*\047)' + uri_nl_string2 = r'(?:"[^"\\]*(?:\\(?:[^\r]|\r\n?)[^"\\]*)*")' + uri_nl_strings = r'(?:%s|%s)' % (uri_nl_string1, uri_nl_string2) + + nl_escaped = r'(?:\\%(nl)s)' % locals() + + space = r'(?:%(spacechar)s|%(comment)s)' % locals() + + ie7hack = r'(?:>/\*\*/)' + + uri = (r'(?:' + r'(?:[^\000-\040"\047()\\\177]*' + r'(?:%(escape)s[^\000-\040"\047()\\\177]*)*)' + r'(?:' + r'(?:%(spacechar)s+|%(nl_escaped)s+)' + r'(?:' + r'(?:[^\000-\040"\047()\\\177]|%(escape)s|%(nl_escaped)s)' + r'[^\000-\040"\047()\\\177]*' + r'(?:%(escape)s[^\000-\040"\047()\\\177]*)*' + r')+' + r')*' + r')') % locals() + + nl_unesc_sub = _re.compile(nl_escaped).sub + + uri_space_sub = _re.compile(( + r'(%(escape)s+)|%(spacechar)s+|%(nl_escaped)s+' + ) % locals()).sub + uri_space_subber = lambda m: m.groups()[0] or '' + + space_sub_simple = _re.compile(( + r'[\r\n\f\040\t;]+|(%(comment)s+)' + ) % locals()).sub + space_sub_banged = _re.compile(( + r'[\r\n\f\040\t;]+|(%(_bang_comment)s+)' + ) % locals()).sub + + post_esc_sub = _re.compile(r'[\r\n\f\t]+').sub + + main_sub = _re.compile(( + r'([^\\"\047u>@\r\n\f\040\t/;:{}]+)' + r'|(?<=[{}(=:>+[,!])(%(space)s+)' + r'|^(%(space)s+)' + r'|(%(space)s+)(?=(([:{});=>+\],!])|$)?)' + r'|;(%(space)s*(?:;%(space)s*)*)(?=(\})?)' + r'|(\{)' + r'|(\})' + r'|(%(strings)s)' + r'|(?<!%(nmchar)s)url\(%(spacechar)s*(' + r'%(uri_nl_strings)s' + r'|%(uri)s' + r')%(spacechar)s*\)' + r'|(@[mM][eE][dD][iI][aA])(?!%(nmchar)s)' + r'|(%(ie7hack)s)(%(space)s*)' + r'|(:[fF][iI][rR][sS][tT]-[lL]' + r'(?:[iI][nN][eE]|[eE][tT][tT][eE][rR]))' + r'(%(space)s*)(?=[{,])' + r'|(%(nl_strings)s)' + r'|(%(escape)s[^\\"\047u>@\r\n\f\040\t/;:{}]*)' + ) % locals()).sub + + # print main_sub.__self__.pattern + + def main_subber(keep_bang_comments): + """ Make main subber """ + in_macie5, in_rule, at_media = [0], [0], [0] + + if keep_bang_comments: + space_sub = space_sub_banged + def space_subber(match): + """ Space|Comment subber """ + if match.lastindex: + group1, group2 = match.group(1, 2) + if group2: + if group1.endswith(r'\*/'): + in_macie5[0] = 1 + else: + in_macie5[0] = 0 + return group1 + elif group1: + if group1.endswith(r'\*/'): + if in_macie5[0]: + return '' + in_macie5[0] = 1 + return r'/*\*/' + elif in_macie5[0]: + in_macie5[0] = 0 + return '/**/' + return '' + else: + space_sub = space_sub_simple + def space_subber(match): + """ Space|Comment subber """ + if match.lastindex: + if match.group(1).endswith(r'\*/'): + if in_macie5[0]: + return '' + in_macie5[0] = 1 + return r'/*\*/' + elif in_macie5[0]: + in_macie5[0] = 0 + return '/**/' + return '' + + def fn_space_post(group): + """ space with token after """ + if group(5) is None or ( + group(6) == ':' and not in_rule[0] and not at_media[0]): + return ' ' + space_sub(space_subber, group(4)) + return space_sub(space_subber, group(4)) + + def fn_semicolon(group): + """ ; handler """ + return ';' + space_sub(space_subber, group(7)) + + def fn_semicolon2(group): + """ ; handler """ + if in_rule[0]: + return space_sub(space_subber, group(7)) + return ';' + space_sub(space_subber, group(7)) + + def fn_open(group): + """ { handler """ + # pylint: disable = W0613 + if at_media[0]: + at_media[0] -= 1 + else: + in_rule[0] = 1 + return '{' + + def fn_close(group): + """ } handler """ + # pylint: disable = W0613 + in_rule[0] = 0 + return '}' + + def fn_media(group): + """ @media handler """ + at_media[0] += 1 + return group(13) + + def fn_ie7hack(group): + """ IE7 Hack handler """ + if not in_rule[0] and not at_media[0]: + in_macie5[0] = 0 + return group(14) + space_sub(space_subber, group(15)) + return '>' + space_sub(space_subber, group(15)) + + table = ( + None, + None, + None, + None, + fn_space_post, # space with token after + fn_space_post, # space with token after + fn_space_post, # space with token after + fn_semicolon, # semicolon + fn_semicolon2, # semicolon + fn_open, # { + fn_close, # } + lambda g: g(11), # string + lambda g: 'url(%s)' % uri_space_sub(uri_space_subber, g(12)), + # url(...) + fn_media, # @media + None, + fn_ie7hack, # ie7hack + None, + lambda g: g(16) + ' ' + space_sub(space_subber, g(17)), + # :first-line|letter followed + # by [{,] (apparently space + # needed for IE6) + lambda g: nl_unesc_sub('', g(18)), # nl_string + lambda g: post_esc_sub(' ', g(19)), # escape + ) + + def func(match): + """ Main subber """ + idx, group = match.lastindex, match.group + if idx > 3: + return table[idx](group) + + # shortcuts for frequent operations below: + elif idx == 1: # not interesting + return group(1) + # else: # space with token before or at the beginning + return space_sub(space_subber, group(idx)) + + return func + + def cssmin(style, keep_bang_comments=False): # pylint: disable = W0621 + """ + Minify CSS. + + :Parameters: + `style` : ``str`` + CSS to minify + + `keep_bang_comments` : ``bool`` + Keep comments starting with an exclamation mark? (``/*!...*/``) + + :Return: Minified style + :Rtype: ``str`` + """ + return main_sub(main_subber(keep_bang_comments), style) + + return cssmin + +cssmin = _make_cssmin() + + +if __name__ == '__main__': + def main(): + """ Main """ + import sys as _sys + keep_bang_comments = ( + '-b' in _sys.argv[1:] + or '-bp' in _sys.argv[1:] + or '-pb' in _sys.argv[1:] + ) + if '-p' in _sys.argv[1:] or '-bp' in _sys.argv[1:] \ + or '-pb' in _sys.argv[1:]: + global cssmin # pylint: disable = W0603 + cssmin = _make_cssmin(python_only=True) + _sys.stdout.write(cssmin( + _sys.stdin.read(), keep_bang_comments=keep_bang_comments + )) + main() diff --git a/compressor/filters/datauri.py b/compressor/filters/datauri.py index 29ae40f..ee67eeb 100644 --- a/compressor/filters/datauri.py +++ b/compressor/filters/datauri.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import os import re import mimetypes @@ -36,10 +37,11 @@ class DataUriFilter(FilterBase): def data_uri_converter(self, matchobj): url = matchobj.group(1).strip(' \'"') - if not url.startswith('data:'): + if not url.startswith('data:') and not url.startswith('//'): path = self.get_file_path(url) if os.stat(path).st_size <= settings.COMPRESS_DATA_URI_MAX_SIZE: - data = b64encode(open(path, 'rb').read()) + with open(path, 'rb') as file: + data = b64encode(file.read()).decode('ascii') return 'url("data:%s;base64,%s")' % ( mimetypes.guess_type(path)[0], data) return 'url("%s")' % url diff --git a/compressor/filters/jsmin/rjsmin.py b/compressor/filters/jsmin/rjsmin.py index ff31b17..6eedf2f 100755 --- a/compressor/filters/jsmin/rjsmin.py +++ b/compressor/filters/jsmin/rjsmin.py @@ -1,8 +1,7 @@ #!/usr/bin/env python # -*- coding: ascii -*- -# flake8: noqa # -# Copyright 2011, 2012 +# Copyright 2011 - 2013 # Andr\xe9 Malo or his licensors, as applicable # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,7 +59,7 @@ __author__ = "Andr\xe9 Malo" __author__ = getattr(__author__, 'decode', lambda x: __author__)('latin-1') __docformat__ = "restructuredtext en" __license__ = "Apache License, Version 2.0" -__version__ = '1.0.5' +__version__ = '1.0.7' __all__ = ['jsmin'] import re as _re @@ -135,10 +134,10 @@ def _make_jsmin(python_only=False): if last is not None: result.append((first, last)) return ''.join(['%s%s%s' % ( - chr(first), - last > first + 1 and '-' or '', - last != first and chr(last) or '' - ) for first, last in result]) + chr(first2), + last2 > first2 + 1 and '-' or '', + last2 != first2 and chr(last2) or '' + ) for first2, last2 in result]) return _re.sub(r'([\000-\040\047])', # for better portability lambda m: '\\%03o' % ord(m.group(1)), (sequentize(result) @@ -172,11 +171,17 @@ def _make_jsmin(python_only=False): id_literal_open = id_literal_(r'[a-zA-Z0-9_${\[(!+-]') id_literal_close = id_literal_(r'[a-zA-Z0-9_$}\])"\047+-]') + dull = r'[^\047"/\000-\040]' + space_sub = _re.compile(( - r'([^\047"/\000-\040]+)' - r'|(%(strings)s[^\047"/\000-\040]*)' - r'|(?:(?<=%(preregex1)s)%(space)s*(%(regex)s[^\047"/\000-\040]*))' - r'|(?:(?<=%(preregex2)s)%(space)s*(%(regex)s[^\047"/\000-\040]*))' + r'(%(dull)s+)' + r'|(%(strings)s%(dull)s*)' + r'|(?<=%(preregex1)s)' + r'%(space)s*(?:%(newline)s%(space)s*)*' + r'(%(regex)s%(dull)s*)' + r'|(?<=%(preregex2)s)' + r'%(space)s*(?:%(newline)s%(space)s)*' + r'(%(regex)s%(dull)s*)' r'|(?<=%(id_literal_close)s)' r'%(space)s*(?:(%(newline)s)%(space)s*)+' r'(?=%(id_literal_open)s)' @@ -186,7 +191,7 @@ def _make_jsmin(python_only=False): r'|%(space)s+' r'|(?:%(newline)s%(space)s*)+' ) % locals()).sub - #print space_sub.__self__.pattern + # print space_sub.__self__.pattern def space_subber(match): """ Substitution callback """ @@ -265,25 +270,28 @@ def jsmin_for_posers(script): return _re.sub( r'([^\047"/\000-\040]+)|((?:(?:\047[^\047\\\r\n]*(?:\\(?:[^\r\n]|\r?' r'\n|\r)[^\047\\\r\n]*)*\047)|(?:"[^"\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|' - r'\r)[^"\\\r\n]*)*"))[^\047"/\000-\040]*)|(?:(?<=[(,=:\[!&|?{};\r\n]' + r'\r)[^"\\\r\n]*)*"))[^\047"/\000-\040]*)|(?<=[(,=:\[!&|?{};\r\n])(?' + r':[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*' + r'(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-\011\013\014\016-\040]|(?:/\*' + r'[^*]*\*+(?:[^/*][^*]*\*+)*/))*)*((?:/(?![\r\n/*])[^/\\\[\r\n]*(?:(' + r'?:\\[^\r\n]|(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/\\\[' + r'\r\n]*)*/)[^\047"/\000-\040]*)|(?<=[\000-#%-,./:-@\[-^`{-~-]return' r')(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/' - r'))*((?:/(?![\r\n/*])[^/\\\[\r\n]*(?:(?:\\[^\r\n]|(?:\[[^\\\]\r\n]*' - r'(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/\\\[\r\n]*)*/)[^\047"/\000-\040]*' - r'))|(?:(?<=[\000-#%-,./:-@\[-^`{-~-]return)(?:[\000-\011\013\014\01' - r'6-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*((?:/(?![\r\n/*])[^/' - r'\\\[\r\n]*(?:(?:\\[^\r\n]|(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]' - r'*)*\]))[^/\\\[\r\n]*)*/)[^\047"/\000-\040]*))|(?<=[^\000-!#%&(*,./' - r':-@\[\\^`{|~])(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/' - r'*][^*]*\*+)*/))*(?:((?:(?://[^\r\n]*)?[\r\n]))(?:[\000-\011\013\01' - r'4\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000-\040"#' - r'%-\047)*,./:-@\\-^`|-~])|(?<=[^\000-#%-,./:-@\[-^`{-~-])((?:[\000-' - r'\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)))+(?=[^' - r'\000-#%-,./:-@\[-^`{-~-])|(?<=\+)((?:[\000-\011\013\014\016-\040]|' - r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)))+(?=\+)|(?<=-)((?:[\000-\011\0' - r'13\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)))+(?=-)|(?:[\0' - r'00-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))+|(?:' - r'(?:(?://[^\r\n]*)?[\r\n])(?:[\000-\011\013\014\016-\040]|(?:/\*[^*' - r']*\*+(?:[^/*][^*]*\*+)*/))*)+', subber, '\n%s\n' % script + r'))*(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-\011\013\014\016-\040]|(?:' + r'/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)))*((?:/(?![\r\n/*])[^/\\\[\r\n]*(?' + r':(?:\\[^\r\n]|(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/' + r'\\\[\r\n]*)*/)[^\047"/\000-\040]*)|(?<=[^\000-!#%&(*,./:-@\[\\^`{|' + r'~])(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)' + r'*/))*(?:((?:(?://[^\r\n]*)?[\r\n]))(?:[\000-\011\013\014\016-\040]' + r'|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000-\040"#%-\047)*,./' + r':-@\\-^`|-~])|(?<=[^\000-#%-,./:-@\[-^`{-~-])((?:[\000-\011\013\01' + r'4\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)))+(?=[^\000-#%-,./:' + r'-@\[-^`{-~-])|(?<=\+)((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*' + r'\*+(?:[^/*][^*]*\*+)*/)))+(?=\+)|(?<=-)((?:[\000-\011\013\014\016-' + r'\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)))+(?=-)|(?:[\000-\011\013' + r'\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))+|(?:(?:(?://[^' + r'\r\n]*)?[\r\n])(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^' + r'/*][^*]*\*+)*/))*)+', subber, '\n%s\n' % script ).strip() diff --git a/compressor/filters/yuglify.py b/compressor/filters/yuglify.py new file mode 100644 index 0000000..07066cc --- /dev/null +++ b/compressor/filters/yuglify.py @@ -0,0 +1,26 @@ +from compressor.conf import settings +from compressor.filters import CompilerFilter + + +class YUglifyFilter(CompilerFilter): + command = "{binary} {args}" + + def __init__(self, *args, **kwargs): + super(YUglifyFilter, self).__init__(*args, **kwargs) + self.command += ' --type=%s' % self.type + + +class YUglifyCSSFilter(YUglifyFilter): + type = 'css' + options = ( + ("binary", settings.COMPRESS_YUGLIFY_BINARY), + ("args", settings.COMPRESS_YUGLIFY_CSS_ARGUMENTS), + ) + + +class YUglifyJSFilter(YUglifyFilter): + type = 'js' + options = ( + ("binary", settings.COMPRESS_YUGLIFY_BINARY), + ("args", settings.COMPRESS_YUGLIFY_JS_ARGUMENTS), + ) diff --git a/compressor/management/commands/compress.py b/compressor/management/commands/compress.py index 1ae0778..6be215e 100644 --- a/compressor/management/commands/compress.py +++ b/compressor/management/commands/compress.py @@ -1,105 +1,33 @@ # flake8: noqa import os import sys -from types import MethodType + from fnmatch import fnmatch from optparse import make_option -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO # noqa - -from django.core.management.base import NoArgsCommand, CommandError -from django.template import (Context, Template, - TemplateDoesNotExist, TemplateSyntaxError) +from django.core.management.base import NoArgsCommand, CommandError +import django.template +from django.template import Context +from django.utils import six from django.utils.datastructures import SortedDict from django.utils.importlib import import_module from django.template.loader import get_template # noqa Leave this in to preload template locations -from django.template.defaulttags import IfNode -from django.template.loader_tags import (ExtendsNode, BlockNode, - BLOCK_CONTEXT_KEY) - -try: - from django.template.loaders.cached import Loader as CachedLoader -except ImportError: - CachedLoader = None # noqa from compressor.cache import get_offline_hexdigest, write_offline_manifest from compressor.conf import settings -from compressor.exceptions import OfflineGenerationError +from compressor.exceptions import (OfflineGenerationError, TemplateSyntaxError, + TemplateDoesNotExist) from compressor.templatetags.compress import CompressorNode - -def patched_render(self, context): - # 'Fake' _render method that just returns the context instead of - # rendering. It also checks whether the first node is an extend node or - # not, to be able to handle complex inheritance chain. - self._render_firstnode = MethodType(patched_render_firstnode, self) - self._render_firstnode(context) - - # Cleanup, uninstall our _render monkeypatch now that it has been called - self._render = self._old_render - return context - - -def patched_render_firstnode(self, context): - # If this template has a ExtendsNode, we want to find out what - # should be put in render_context to make the {% block ... %} - # tags work. - # - # We can't fully render the base template(s) (we don't have the - # full context vars - only what's necessary to render the compress - # nodes!), therefore we hack the ExtendsNode we found, patching - # its get_parent method so that rendering the ExtendsNode only - # gives us the blocks content without doing any actual rendering. - extra_context = {} +if six.PY3: + # there is an 'io' module in python 2.6+, but io.StringIO does not + # accept regular strings, just unicode objects + from io import StringIO +else: try: - firstnode = self.nodelist[0] - except IndexError: - firstnode = None - if isinstance(firstnode, ExtendsNode): - firstnode._log = self._log - firstnode._log_verbosity = self._log_verbosity - firstnode._old_get_parent = firstnode.get_parent - firstnode.get_parent = MethodType(patched_get_parent, firstnode) - try: - extra_context = firstnode.render(context) - context.render_context = extra_context.render_context - # We aren't rendering {% block %} tags, but we want - # {{ block.super }} inside {% compress %} inside {% block %}s to - # work. Therefore, we need to pop() the last block context for - # each block name, to emulate what would have been done if the - # {% block %} had been fully rendered. - for blockname in firstnode.blocks.keys(): - context.render_context[BLOCK_CONTEXT_KEY].pop(blockname) - except (IOError, TemplateSyntaxError, TemplateDoesNotExist): - # That first node we are trying to render might cause more errors - # that we didn't catch when simply creating a Template instance - # above, so we need to catch that (and ignore it, just like above) - # as well. - if self._log_verbosity > 0: - self._log.write("Caught error when rendering extend node from " - "template %s\n" % getattr(self, 'name', self)) - return None - finally: - # Cleanup, uninstall our get_parent monkeypatch now that it has been called - firstnode.get_parent = firstnode._old_get_parent - return extra_context - - -def patched_get_parent(self, context): - # Patch template returned by extendsnode's get_parent to make sure their - # _render method is just returning the context instead of actually - # rendering stuff. - # In addition, this follows the inheritance chain by looking if the first - # node of the template is an extend node itself. - compiled_template = self._old_get_parent(context) - compiled_template._log = self._log - compiled_template._log_verbosity = self._log_verbosity - compiled_template._old_render = compiled_template._render - compiled_template._render = MethodType(patched_render, compiled_template) - return compiled_template + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO class Command(NoArgsCommand): @@ -117,6 +45,9 @@ class Command(NoArgsCommand): "(which defaults to STATIC_ROOT). Be aware that using this " "can lead to infinite recursion if a link points to a parent " "directory of itself.", dest='follow_links'), + make_option('--engine', default="django", action="store", + help="Specifies the templating engine. jinja2 or django", + dest="engine"), ) requires_model_validation = False @@ -134,7 +65,7 @@ class Command(NoArgsCommand): # Force django to calculate template_source_loaders from # TEMPLATE_LOADERS settings, by asking to find a dummy template source, name = finder_func('test') - except TemplateDoesNotExist: + except django.template.TemplateDoesNotExist: pass # Reload template_source_loaders now that it has been calculated ; # it should contain the list of valid, instanciated template loaders @@ -151,13 +82,28 @@ class Command(NoArgsCommand): # ) # The loaders will return django.template.loaders.filesystem.Loader # and django.template.loaders.app_directories.Loader + # The cached Loader and similar ones include a 'loaders' attribute + # so we look for that. for loader in template_source_loaders: - if CachedLoader is not None and isinstance(loader, CachedLoader): + if hasattr(loader, 'loaders'): loaders.extend(loader.loaders) else: loaders.append(loader) return loaders + def __get_parser(self, engine): + if engine == "jinja2": + from compressor.offline.jinja2 import Jinja2Parser + env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT() + parser = Jinja2Parser(charset=settings.FILE_CHARSET, env=env) + elif engine == "django": + from compressor.offline.django import DjangoParser + parser = DjangoParser(charset=settings.FILE_CHARSET) + else: + raise OfflineGenerationError("Invalid templating engine specified.") + + return parser + def compress(self, log=None, **options): """ Searches templates containing 'compress' nodes and compresses them @@ -210,20 +156,18 @@ class Command(NoArgsCommand): if verbosity > 1: log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n") + engine = options.get("engine", "django") + parser = self.__get_parser(engine) + compressor_nodes = SortedDict() for template_name in templates: try: - template_file = open(template_name) - try: - template = Template(template_file.read().decode( - settings.FILE_CHARSET)) - finally: - template_file.close() + template = parser.parse(template_name) except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) continue - except TemplateSyntaxError, e: # broken template -> ignore + except TemplateSyntaxError as e: # broken template -> ignore if verbosity > 0: log.write("Invalid template %s: %s\n" % (template_name, e)) continue @@ -235,7 +179,13 @@ class Command(NoArgsCommand): if verbosity > 0: log.write("UnicodeDecodeError while trying to read " "template %s\n" % template_name) - nodes = list(self.walk_nodes(template)) + try: + nodes = list(parser.walk_nodes(template)) + except (TemplateDoesNotExist, TemplateSyntaxError) as e: + # Could be an error in some base template + if verbosity > 0: + log.write("Error parsing template %s: %s\n" % (template_name, e)) + continue if nodes: template.template_name = template_name compressor_nodes.setdefault(template, []).extend(nodes) @@ -255,27 +205,28 @@ class Command(NoArgsCommand): count = 0 results = [] offline_manifest = SortedDict() - for template, nodes in compressor_nodes.iteritems(): - context = Context(settings.COMPRESS_OFFLINE_CONTEXT) + init_context = parser.get_init_context(settings.COMPRESS_OFFLINE_CONTEXT) + + for template, nodes in compressor_nodes.items(): + context = Context(init_context) template._log = log template._log_verbosity = verbosity - template._render_firstnode = MethodType(patched_render_firstnode, template) - extra_context = template._render_firstnode(context) - if extra_context is None: - # Something is wrong - ignore this template + + if not parser.process_template(template, context): continue + for node in nodes: context.push() - if extra_context and node._block_name: - # Give a block context to the node if it was found inside - # a {% block %}. - context['block'] = context.render_context[BLOCK_CONTEXT_KEY].get_block(node._block_name) - if context['block']: - context['block'].context = context - key = get_offline_hexdigest(node.nodelist.render(context)) + parser.process_node(template, context, node) + rendered = parser.render_nodelist(template, context, node) + key = get_offline_hexdigest(rendered) + + if key in offline_manifest: + continue + try: - result = node.render(context, forced=True) - except Exception, e: + result = parser.render_node(template, context, node) + except Exception as e: raise CommandError("An error occured during rendering %s: " "%s" % (template.template_name, e)) offline_manifest[key] = result @@ -289,23 +240,6 @@ class Command(NoArgsCommand): (count, len(compressor_nodes))) return count, results - def get_nodelist(self, node): - # Check if node is an ```if``` switch with true and false branches - if hasattr(node, 'nodelist_true') and hasattr(node, 'nodelist_false'): - return node.nodelist_true + node.nodelist_false - return getattr(node, "nodelist", []) - - def walk_nodes(self, node, block_name=None): - for node in self.get_nodelist(node): - if isinstance(node, BlockNode): - block_name = node.name - if isinstance(node, CompressorNode) and node.is_offline_compression_enabled(forced=True): - node._block_name = block_name - yield node - else: - for node in self.walk_nodes(node, block_name=block_name): - yield node - def handle_extensions(self, extensions=('html',)): """ organizes multiple extensions that are separated with commas or @@ -331,7 +265,7 @@ class Command(NoArgsCommand): if not settings.COMPRESS_ENABLED and not options.get("force"): raise CommandError( "Compressor is disabled. Set the COMPRESS_ENABLED " - "settting or use --force to override.") + "setting or use --force to override.") if not settings.COMPRESS_OFFLINE: if not options.get("force"): raise CommandError( diff --git a/compressor/management/commands/mtime_cache.py b/compressor/management/commands/mtime_cache.py index bfea571..e96f004 100644 --- a/compressor/management/commands/mtime_cache.py +++ b/compressor/management/commands/mtime_cache.py @@ -74,9 +74,9 @@ class Command(NoArgsCommand): if keys_to_delete: cache.delete_many(list(keys_to_delete)) - print "Deleted mtimes of %d files from the cache." % len(keys_to_delete) + print("Deleted mtimes of %d files from the cache." % len(keys_to_delete)) if files_to_add: for filename in files_to_add: get_mtime(filename) - print "Added mtimes of %d files to cache." % len(files_to_add) + print("Added mtimes of %d files to cache." % len(files_to_add)) diff --git a/compressor/offline/__init__.py b/compressor/offline/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/compressor/offline/__init__.py diff --git a/compressor/offline/django.py b/compressor/offline/django.py new file mode 100644 index 0000000..6541471 --- /dev/null +++ b/compressor/offline/django.py @@ -0,0 +1,143 @@ +from __future__ import absolute_import +import io +from copy import copy + +from django import template +from django.conf import settings +from django.template import Template +from django.template import Context +from django.template.base import Node, VariableNode, TextNode, NodeList +from django.template.defaulttags import IfNode +from django.template.loader_tags import ExtendsNode, BlockNode, BlockContext + + +from compressor.exceptions import TemplateSyntaxError, TemplateDoesNotExist +from compressor.templatetags.compress import CompressorNode + + +def handle_extendsnode(extendsnode, block_context=None): + """Create a copy of Node tree of a derived template replacing + all blocks tags with the nodes of appropriate blocks. + Also handles {{ block.super }} tags. + """ + if block_context is None: + block_context = BlockContext() + blocks = dict((n.name, n) for n in + extendsnode.nodelist.get_nodes_by_type(BlockNode)) + block_context.add_blocks(blocks) + + context = Context(settings.COMPRESS_OFFLINE_CONTEXT) + compiled_parent = extendsnode.get_parent(context) + parent_nodelist = compiled_parent.nodelist + # If the parent template has an ExtendsNode it is not the root. + for node in parent_nodelist: + # The ExtendsNode has to be the first non-text node. + if not isinstance(node, TextNode): + if isinstance(node, ExtendsNode): + return handle_extendsnode(node, block_context) + break + # Add blocks of the root template to block context. + blocks = dict((n.name, n) for n in + parent_nodelist.get_nodes_by_type(BlockNode)) + block_context.add_blocks(blocks) + + block_stack = [] + new_nodelist = remove_block_nodes(parent_nodelist, block_stack, block_context) + return new_nodelist + + +def remove_block_nodes(nodelist, block_stack, block_context): + new_nodelist = NodeList() + for node in nodelist: + if isinstance(node, VariableNode): + var_name = node.filter_expression.token.strip() + if var_name == 'block.super': + if not block_stack: + continue + node = block_context.get_block(block_stack[-1].name) + if isinstance(node, BlockNode): + expanded_block = expand_blocknode(node, block_stack, block_context) + new_nodelist.extend(expanded_block) + else: + # IfNode has nodelist as a @property so we can not modify it + if isinstance(node, IfNode): + node = copy(node) + for i, (condition, sub_nodelist) in enumerate(node.conditions_nodelists): + sub_nodelist = remove_block_nodes(sub_nodelist, block_stack, block_context) + node.conditions_nodelists[i] = (condition, sub_nodelist) + else: + for attr in node.child_nodelists: + sub_nodelist = getattr(node, attr, None) + if sub_nodelist: + sub_nodelist = remove_block_nodes(sub_nodelist, block_stack, block_context) + node = copy(node) + setattr(node, attr, sub_nodelist) + new_nodelist.append(node) + return new_nodelist + + +def expand_blocknode(node, block_stack, block_context): + popped_block = block = block_context.pop(node.name) + if block is None: + block = node + block_stack.append(block) + expanded_nodelist = remove_block_nodes(block.nodelist, block_stack, block_context) + block_stack.pop() + if popped_block is not None: + block_context.push(node.name, popped_block) + return expanded_nodelist + + +class DjangoParser(object): + def __init__(self, charset): + self.charset = charset + + def parse(self, template_name): + with io.open(template_name, mode='rb') as file: + try: + return Template(file.read().decode(self.charset)) + except template.TemplateSyntaxError as e: + raise TemplateSyntaxError(str(e)) + except template.TemplateDoesNotExist as e: + raise TemplateDoesNotExist(str(e)) + + def process_template(self, template, context): + return True + + def get_init_context(self, offline_context): + return offline_context + + def process_node(self, template, context, node): + pass + + def render_nodelist(self, template, context, node): + return node.nodelist.render(context) + + def render_node(self, template, context, node): + return node.render(context, forced=True) + + def get_nodelist(self, node): + if isinstance(node, ExtendsNode): + try: + return handle_extendsnode(node) + except template.TemplateSyntaxError as e: + raise TemplateSyntaxError(str(e)) + except template.TemplateDoesNotExist as e: + raise TemplateDoesNotExist(str(e)) + + # Check if node is an ```if``` switch with true and false branches + nodelist = [] + if isinstance(node, Node): + for attr in node.child_nodelists: + nodelist += getattr(node, attr, []) + else: + nodelist = getattr(node, 'nodelist', []) + return nodelist + + def walk_nodes(self, node): + for node in self.get_nodelist(node): + if isinstance(node, CompressorNode) and node.is_offline_compression_enabled(forced=True): + yield node + else: + for node in self.walk_nodes(node): + yield node diff --git a/compressor/offline/jinja2.py b/compressor/offline/jinja2.py new file mode 100644 index 0000000..feee818 --- /dev/null +++ b/compressor/offline/jinja2.py @@ -0,0 +1,125 @@ +from __future__ import absolute_import +import io + +import jinja2 +import jinja2.ext +from jinja2 import nodes +from jinja2.ext import Extension +from jinja2.nodes import CallBlock, Call, ExtensionAttribute + +from compressor.exceptions import TemplateSyntaxError, TemplateDoesNotExist + + +def flatten_context(context): + if hasattr(context, 'dicts'): + context_dict = {} + + for d in context.dicts: + context_dict.update(d) + + return context_dict + + return context + + +class SpacelessExtension(Extension): + """ + Functional "spaceless" extension equivalent to Django's. + + See: https://github.com/django/django/blob/master/django/template/defaulttags.py + """ + + tags = set(['spaceless']) + + def parse(self, parser): + lineno = next(parser.stream).lineno + body = parser.parse_statements(['name:endspaceless'], drop_needle=True) + + return nodes.CallBlock(self.call_method('_spaceless', []), + [], [], body).set_lineno(lineno) + + def _spaceless(self, caller): + from django.utils.html import strip_spaces_between_tags + + return strip_spaces_between_tags(caller().strip()) + + +def url_for(mod, filename): + """ + Incomplete emulation of Flask's url_for. + """ + from django.contrib.staticfiles.templatetags import staticfiles + + if mod == "static": + return staticfiles.static(filename) + + return "" + + +class Jinja2Parser(object): + COMPRESSOR_ID = 'compressor.contrib.jinja2ext.CompressorExtension' + + def __init__(self, charset, env): + self.charset = charset + self.env = env + + def parse(self, template_name): + with io.open(template_name, mode='rb') as file: + try: + template = self.env.parse(file.read().decode(self.charset)) + except jinja2.TemplateSyntaxError as e: + raise TemplateSyntaxError(str(e)) + except jinja2.TemplateNotFound as e: + raise TemplateDoesNotExist(str(e)) + + return template + + def process_template(self, template, context): + return True + + def get_init_context(self, offline_context): + # Don't need to add filters and tests to the context, as Jinja2 will + # automatically look for them in self.env.filters and self.env.tests. + # This is tested by test_complex and test_templatetag. + + # Allow offline context to override the globals. + context = self.env.globals.copy() + context.update(offline_context) + + return context + + def process_node(self, template, context, node): + pass + + def _render_nodes(self, template, context, nodes): + compiled_node = self.env.compile(jinja2.nodes.Template(nodes)) + template = jinja2.Template.from_code(self.env, compiled_node, {}) + flat_context = flatten_context(context) + + return template.render(flat_context) + + def render_nodelist(self, template, context, node): + return self._render_nodes(template, context, node.body) + + def render_node(self, template, context, node): + return self._render_nodes(template, context, [node]) + + def get_nodelist(self, node): + body = getattr(node, "body", getattr(node, "nodes", [])) + + if isinstance(node, jinja2.nodes.If): + return body + node.else_ + + return body + + def walk_nodes(self, node, block_name=None): + for node in self.get_nodelist(node): + if (isinstance(node, CallBlock) and + isinstance(node.call, Call) and + isinstance(node.call.node, ExtensionAttribute) and + node.call.node.identifier == self.COMPRESSOR_ID): + node.call.node.name = '_compress_forced' + yield node + else: + for node in self.walk_nodes(node, block_name=block_name): + yield node diff --git a/compressor/parser/__init__.py b/compressor/parser/__init__.py index bc8c18c..a3fe78f 100644 --- a/compressor/parser/__init__.py +++ b/compressor/parser/__init__.py @@ -1,3 +1,4 @@ +from django.utils import six from django.utils.functional import LazyObject from django.utils.importlib import import_module @@ -11,8 +12,9 @@ from compressor.parser.html5lib import Html5LibParser # noqa class AutoSelectParser(LazyObject): options = ( - ('lxml.html', LxmlParser), # lxml, extremely fast - ('HTMLParser', HtmlParser), # fast and part of the Python stdlib + # TODO: make lxml.html parser first again + (six.moves.html_parser.__name__, HtmlParser), # fast and part of the Python stdlib + ('lxml.html', LxmlParser), # lxml, extremely fast ) def __init__(self, content): diff --git a/compressor/parser/beautifulsoup.py b/compressor/parser/beautifulsoup.py index 498cde8..d143df4 100644 --- a/compressor/parser/beautifulsoup.py +++ b/compressor/parser/beautifulsoup.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from django.core.exceptions import ImproperlyConfigured -from django.utils.encoding import smart_unicode +from django.utils import six +from django.utils.encoding import smart_text from compressor.exceptions import ParserError from compressor.parser import ParserBase @@ -12,18 +13,27 @@ class BeautifulSoupParser(ParserBase): @cached_property def soup(self): try: - from BeautifulSoup import BeautifulSoup + if six.PY3: + from bs4 import BeautifulSoup + else: + from BeautifulSoup import BeautifulSoup return BeautifulSoup(self.content) - except ImportError, err: + except ImportError as err: raise ImproperlyConfigured("Error while importing BeautifulSoup: %s" % err) - except Exception, err: + except Exception as err: raise ParserError("Error while initializing Parser: %s" % err) def css_elems(self): - return self.soup.findAll({'link': True, 'style': True}) + if six.PY3: + return self.soup.find_all({'link': True, 'style': True}) + else: + return self.soup.findAll({'link': True, 'style': True}) def js_elems(self): - return self.soup.findAll('script') + if six.PY3: + return self.soup.find_all('script') + else: + return self.soup.findAll('script') def elem_attribs(self, elem): return dict(elem.attrs) @@ -35,4 +45,4 @@ class BeautifulSoupParser(ParserBase): return elem.name def elem_str(self, elem): - return smart_unicode(elem) + return smart_text(elem) diff --git a/compressor/parser/default_htmlparser.py b/compressor/parser/default_htmlparser.py index 8425d77..80272cb 100644 --- a/compressor/parser/default_htmlparser.py +++ b/compressor/parser/default_htmlparser.py @@ -1,13 +1,13 @@ -from HTMLParser import HTMLParser -from django.utils.encoding import smart_unicode +from django.utils import six +from django.utils.encoding import smart_text + from compressor.exceptions import ParserError from compressor.parser import ParserBase -class DefaultHtmlParser(ParserBase, HTMLParser): - +class DefaultHtmlParser(ParserBase, six.moves.html_parser.HTMLParser): def __init__(self, content): - HTMLParser.__init__(self) + six.moves.html_parser.HTMLParser.__init__(self) self.content = content self._css_elems = [] self._js_elems = [] @@ -15,7 +15,7 @@ class DefaultHtmlParser(ParserBase, HTMLParser): try: self.feed(self.content) self.close() - except Exception, err: + except Exception as err: lineno = err.lineno line = self.content.splitlines()[lineno] raise ParserError("Error while initializing HtmlParser: %s (line: %s)" % (err, repr(line))) @@ -65,7 +65,7 @@ class DefaultHtmlParser(ParserBase, HTMLParser): return elem['attrs_dict'] def elem_content(self, elem): - return smart_unicode(elem['text']) + return smart_text(elem['text']) def elem_str(self, elem): tag = {} diff --git a/compressor/parser/html5lib.py b/compressor/parser/html5lib.py index 7fee590..b1d0948 100644 --- a/compressor/parser/html5lib.py +++ b/compressor/parser/html5lib.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from django.utils.encoding import smart_unicode from django.core.exceptions import ImproperlyConfigured +from django.utils.encoding import smart_text from compressor.exceptions import ParserError from compressor.parser import ParserBase @@ -15,42 +15,45 @@ class Html5LibParser(ParserBase): self.html5lib = html5lib def _serialize(self, elem): - fragment = self.html5lib.treebuilders.simpletree.DocumentFragment() - fragment.appendChild(elem) - return self.html5lib.serialize(fragment, - quote_attr_values=True, omit_optional_tags=False) + return self.html5lib.serialize( + elem, tree="etree", quote_attr_values=True, + omit_optional_tags=False, use_trailing_solidus=True, + ) def _find(self, *names): - for node in self.html.childNodes: - if node.type == 5 and node.name in names: - yield node + for elem in self.html: + if elem.tag in names: + yield elem @cached_property def html(self): try: - return self.html5lib.parseFragment(self.content) - except ImportError, err: + return self.html5lib.parseFragment(self.content, treebuilder="etree") + except ImportError as err: raise ImproperlyConfigured("Error while importing html5lib: %s" % err) - except Exception, err: + except Exception as err: raise ParserError("Error while initializing Parser: %s" % err) def css_elems(self): - return self._find('style', 'link') + return self._find('{http://www.w3.org/1999/xhtml}link', + '{http://www.w3.org/1999/xhtml}style') def js_elems(self): - return self._find('script') + return self._find('{http://www.w3.org/1999/xhtml}script') def elem_attribs(self, elem): - return elem.attributes + return elem.attrib def elem_content(self, elem): - return elem.childNodes[0].value + return smart_text(elem.text) def elem_name(self, elem): - return elem.name + if '}' in elem.tag: + return elem.tag.split('}')[1] + return elem.tag def elem_str(self, elem): # This method serializes HTML in a way that does not pass all tests. # However, this method is only called in tests anyway, so it doesn't # really matter. - return smart_unicode(self._serialize(elem)) + return smart_text(self._serialize(elem)) diff --git a/compressor/parser/lxml.py b/compressor/parser/lxml.py index 7bbb561..64a8fcb 100644 --- a/compressor/parser/lxml.py +++ b/compressor/parser/lxml.py @@ -1,6 +1,8 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals + from django.core.exceptions import ImproperlyConfigured -from django.utils.encoding import smart_unicode +from django.utils import six +from django.utils.encoding import smart_text from compressor.exceptions import ParserError from compressor.parser import ParserBase @@ -8,28 +10,50 @@ from compressor.utils.decorators import cached_property class LxmlParser(ParserBase): - + """ + LxmlParser will use `lxml.html` parser to parse rendered contents of + {% compress %} tag. Under python 2 it will also try to use beautiful + soup parser in case of any problems with encoding. + """ def __init__(self, content): try: - from lxml.html import fromstring, soupparser + from lxml.html import fromstring from lxml.etree import tostring - self.fromstring = fromstring - self.soupparser = soupparser - self.tostring = tostring - except ImportError, err: + except ImportError as err: raise ImproperlyConfigured("Error while importing lxml: %s" % err) - except Exception, err: - raise ParserError("Error while initializing Parser: %s" % err) + except Exception as err: + raise ParserError("Error while initializing parser: %s" % err) + + if not six.PY3: + # soupparser uses Beautiful Soup 3 which does not run on python 3.x + try: + from lxml.html import soupparser + except ImportError as err: + soupparser = None + except Exception as err: + raise ParserError("Error while initializing parser: %s" % err) + else: + soupparser = None + + self.soupparser = soupparser + self.fromstring = fromstring + self.tostring = tostring super(LxmlParser, self).__init__(content) @cached_property def tree(self): + """ + Document tree. + """ content = '<root>%s</root>' % self.content tree = self.fromstring(content) try: - self.tostring(tree, encoding=unicode) + self.tostring(tree, encoding=six.text_type) except UnicodeDecodeError: - tree = self.soupparser.fromstring(content) + if self.soupparser: # use soup parser on python 2 + tree = self.soupparser.fromstring(content) + else: # raise an error on python 3 + raise return tree def css_elems(self): @@ -43,14 +67,14 @@ class LxmlParser(ParserBase): return elem.attrib def elem_content(self, elem): - return smart_unicode(elem.text) + return smart_text(elem.text) def elem_name(self, elem): return elem.tag def elem_str(self, elem): - elem_as_string = smart_unicode( - self.tostring(elem, method='html', encoding=unicode)) + elem_as_string = smart_text( + self.tostring(elem, method='html', encoding=six.text_type)) if elem.tag == 'link': # This makes testcases happy return elem_as_string.replace('>', ' />') diff --git a/compressor/storage.py b/compressor/storage.py index be9b066..16419a8 100644 --- a/compressor/storage.py +++ b/compressor/storage.py @@ -1,7 +1,9 @@ +from __future__ import unicode_literals import errno import gzip -from os import path +import os from datetime import datetime +import time from django.core.files.storage import FileSystemStorage, get_storage_class from django.utils.functional import LazyObject, SimpleLazyObject @@ -26,13 +28,13 @@ class CompressorFileStorage(FileSystemStorage): *args, **kwargs) def accessed_time(self, name): - return datetime.fromtimestamp(path.getatime(self.path(name))) + return datetime.fromtimestamp(os.path.getatime(self.path(name))) def created_time(self, name): - return datetime.fromtimestamp(path.getctime(self.path(name))) + return datetime.fromtimestamp(os.path.getctime(self.path(name))) def modified_time(self, name): - return datetime.fromtimestamp(path.getmtime(self.path(name))) + return datetime.fromtimestamp(os.path.getmtime(self.path(name))) def get_available_name(self, name): """ @@ -49,7 +51,7 @@ class CompressorFileStorage(FileSystemStorage): """ try: super(CompressorFileStorage, self).delete(name) - except OSError, e: + except OSError as e: if e.errno != errno.ENOENT: raise @@ -65,9 +67,25 @@ class GzipCompressorFileStorage(CompressorFileStorage): """ def save(self, filename, content): filename = super(GzipCompressorFileStorage, self).save(filename, content) - out = gzip.open(u'%s.gz' % self.path(filename), 'wb') - out.writelines(open(self.path(filename), 'rb')) - out.close() + orig_path = self.path(filename) + compressed_path = '%s.gz' % orig_path + + f_in = open(orig_path, 'rb') + f_out = open(compressed_path, 'wb') + try: + f_out = gzip.GzipFile(fileobj=f_out) + f_out.write(f_in.read()) + finally: + f_out.close() + f_in.close() + # Ensure the file timestamps match. + # os.stat() returns nanosecond resolution on Linux, but os.utime() + # only sets microsecond resolution. Set times on both files to + # ensure they are equal. + stamp = time.time() + os.utime(orig_path, (stamp, stamp)) + os.utime(compressed_path, (stamp, stamp)) + return filename diff --git a/compressor/templatetags/compress.py b/compressor/templatetags/compress.py index 870668a..a45f454 100644 --- a/compressor/templatetags/compress.py +++ b/compressor/templatetags/compress.py @@ -1,5 +1,6 @@ from django import template from django.core.exceptions import ImproperlyConfigured +from django.utils import six from compressor.cache import (cache_get, cache_set, get_offline_hexdigest, get_offline_manifest, get_templatetag_cachekey) @@ -50,7 +51,7 @@ class CompressorMixin(object): Check if offline compression is enabled or forced Defaults to just checking the settings and forced argument, - but can be overriden to completely disable compression for + but can be overridden to completely disable compression for a subclass, for instance. """ return (settings.COMPRESS_ENABLED and @@ -107,6 +108,7 @@ class CompressorMixin(object): rendered_output = self.render_output(compressor, mode, forced=forced) if cache_key: cache_set(cache_key, rendered_output) + assert isinstance(rendered_output, six.string_types) return rendered_output except Exception: if settings.DEBUG or forced: @@ -199,7 +201,7 @@ def compress(parser, token): if len(args) >= 3: mode = args[2] - if not mode in OUTPUT_MODES: + if mode not in OUTPUT_MODES: raise template.TemplateSyntaxError( "%r's second argument must be '%s' or '%s'." % (args[0], OUTPUT_FILE, OUTPUT_INLINE)) diff --git a/compressor/test_settings.py b/compressor/test_settings.py index 0e8a768..a5abf92 100644 --- a/compressor/test_settings.py +++ b/compressor/test_settings.py @@ -5,18 +5,17 @@ TEST_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'tests') COMPRESS_CACHE_BACKEND = 'locmem://' -if django.VERSION[:2] >= (1, 3): - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', - } +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', } -else: - DATABASE_ENGINE = 'sqlite3' +} INSTALLED_APPS = [ 'compressor', + 'coffin', + 'jingo', ] STATIC_URL = '/static/' @@ -31,6 +30,11 @@ TEMPLATE_DIRS = ( os.path.join(TEST_DIR, 'test_templates'), ) -TEST_RUNNER = 'discover_runner.DiscoverRunner' +if django.VERSION[:2] < (1, 6): + TEST_RUNNER = 'discover_runner.DiscoverRunner' SECRET_KEY = "iufoj=mibkpdz*%bob952x(%49rqgv8gg45k36kjcg76&-y5=!" + +PASSWORD_HASHERS = ( + 'django.contrib.auth.hashers.UnsaltedMD5PasswordHasher', +) diff --git a/compressor/tests/precompiler.py b/compressor/tests/precompiler.py index 4c01964..059a322 100644 --- a/compressor/tests/precompiler.py +++ b/compressor/tests/precompiler.py @@ -28,7 +28,7 @@ def main(): with open(options.outfile, 'w') as f: f.write(content) else: - print content + print(content) if __name__ == '__main__': diff --git a/compressor/tests/test_base.py b/compressor/tests/test_base.py index 8678e32..46b1d91 100644 --- a/compressor/tests/test_base.py +++ b/compressor/tests/test_base.py @@ -1,11 +1,16 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals import os import re -from BeautifulSoup import BeautifulSoup +try: + from bs4 import BeautifulSoup +except ImportError: + from BeautifulSoup import BeautifulSoup +from django.utils import six from django.core.cache.backends import locmem -from django.test import TestCase +from django.test import SimpleTestCase +from django.test.utils import override_settings from compressor.base import SOURCE_HUNK, SOURCE_FILE from compressor.conf import settings @@ -14,15 +19,24 @@ from compressor.js import JsCompressor from compressor.exceptions import FilterDoesNotExist +def make_soup(markup): + # we use html.parser instead of lxml because it doesn't work on python 3.3 + if six.PY3: + return BeautifulSoup(markup, 'html.parser') + else: + return BeautifulSoup(markup) + + def css_tag(href, **kwargs): rendered_attrs = ''.join(['%s="%s" ' % (k, v) for k, v in kwargs.items()]) - template = u'<link rel="stylesheet" href="%s" type="text/css" %s/>' + template = '<link rel="stylesheet" href="%s" type="text/css" %s/>' return template % (href, rendered_attrs) class TestPrecompiler(object): """A filter whose output is always the string 'OUTPUT' """ - def __init__(self, content, attrs, filter_type=None, filename=None): + def __init__(self, content, attrs, filter_type=None, filename=None, + charset=None): pass def input(self, **kwargs): @@ -32,11 +46,11 @@ class TestPrecompiler(object): test_dir = os.path.abspath(os.path.join(os.path.dirname(__file__))) -class CompressorTestCase(TestCase): +class CompressorTestCase(SimpleTestCase): def setUp(self): settings.COMPRESS_ENABLED = True - settings.COMPRESS_PRECOMPILERS = {} + settings.COMPRESS_PRECOMPILERS = () settings.COMPRESS_DEBUG_TOGGLE = 'nocompress' self.css = """\ <link rel="stylesheet" href="/static/css/one.css" type="text/css" /> @@ -49,22 +63,52 @@ class CompressorTestCase(TestCase): <script type="text/javascript">obj.value = "value";</script>""" self.js_node = JsCompressor(self.js) + def assertEqualCollapsed(self, a, b): + """ + assertEqual with internal newlines collapsed to single, and + trailing whitespace removed. + """ + collapse = lambda x: re.sub(r'\n+', '\n', x).rstrip() + self.assertEqual(collapse(a), collapse(b)) + + def assertEqualSplits(self, a, b): + """ + assertEqual for splits, particularly ignoring the presence of + a trailing newline on the content. + """ + mangle = lambda split: [(x[0], x[1], x[2], x[3].rstrip()) for x in split] + self.assertEqual(mangle(a), mangle(b)) + def test_css_split(self): out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'one.css'), u'css/one.css', u'<link rel="stylesheet" href="/static/css/one.css" type="text/css" />'), - (SOURCE_HUNK, u'p { border:5px solid green;}', None, u'<style type="text/css">p { border:5px solid green;}</style>'), - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'two.css'), u'css/two.css', u'<link rel="stylesheet" href="/static/css/two.css" type="text/css" />'), + ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'css', 'one.css'), + 'css/one.css', '<link rel="stylesheet" href="/static/css/one.css" type="text/css" />', + ), + ( + SOURCE_HUNK, + 'p { border:5px solid green;}', + None, + '<style type="text/css">p { border:5px solid green;}</style>', + ), + ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'css', 'two.css'), + 'css/two.css', + '<link rel="stylesheet" href="/static/css/two.css" type="text/css" />', + ), ] split = self.css_node.split_contents() split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split] - self.assertEqual(out, split) + self.assertEqualSplits(split, out) def test_css_hunks(self): - out = ['body { background:#990; }', u'p { border:5px solid green;}', 'body { color:#fff; }'] + out = ['body { background:#990; }', 'p { border:5px solid green;}', 'body { color:#fff; }'] self.assertEqual(out, list(self.css_node.hunks())) def test_css_output(self): - out = u'body { background:#990; }\np { border:5px solid green;}\nbody { color:#fff; }' + out = 'body { background:#990; }\np { border:5px solid green;}\nbody { color:#fff; }' hunks = '\n'.join([h for h in self.css_node.hunks()]) self.assertEqual(out, hunks) @@ -76,7 +120,7 @@ class CompressorTestCase(TestCase): def test_css_return_if_off(self): settings.COMPRESS_ENABLED = False - self.assertEqual(self.css, self.css_node.output()) + self.assertEqualCollapsed(self.css, self.css_node.output()) def test_cachekey(self): is_cachekey = re.compile(r'\w{12}') @@ -89,90 +133,83 @@ class CompressorTestCase(TestCase): def test_js_split(self): out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'js', u'one.js'), u'js/one.js', '<script src="/static/js/one.js" type="text/javascript"></script>'), - (SOURCE_HUNK, u'obj.value = "value";', None, '<script type="text/javascript">obj.value = "value";</script>'), + ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'js', 'one.js'), + 'js/one.js', + '<script src="/static/js/one.js" type="text/javascript"></script>', + ), + ( + SOURCE_HUNK, + 'obj.value = "value";', + None, + '<script type="text/javascript">obj.value = "value";</script>', + ), ] split = self.js_node.split_contents() split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split] - self.assertEqual(out, split) + self.assertEqualSplits(split, out) def test_js_hunks(self): - out = ['obj = {};', u'obj.value = "value";'] + out = ['obj = {};', 'obj.value = "value";'] self.assertEqual(out, list(self.js_node.hunks())) def test_js_output(self): - out = u'<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' + out = '<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' self.assertEqual(out, self.js_node.output()) def test_js_override_url(self): - self.js_node.context.update({'url': u'This is not a url, just a text'}) - out = u'<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' + self.js_node.context.update({'url': 'This is not a url, just a text'}) + out = '<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' self.assertEqual(out, self.js_node.output()) def test_css_override_url(self): - self.css_node.context.update({'url': u'This is not a url, just a text'}) + self.css_node.context.update({'url': 'This is not a url, just a text'}) output = css_tag('/static/CACHE/css/e41ba2cc6982.css') self.assertEqual(output, self.css_node.output().strip()) + @override_settings(COMPRESS_PRECOMPILERS=(), COMPRESS_ENABLED=False) def test_js_return_if_off(self): - try: - enabled = settings.COMPRESS_ENABLED - precompilers = settings.COMPRESS_PRECOMPILERS - settings.COMPRESS_ENABLED = False - settings.COMPRESS_PRECOMPILERS = {} - self.assertEqual(self.js, self.js_node.output()) - finally: - settings.COMPRESS_ENABLED = enabled - settings.COMPRESS_PRECOMPILERS = precompilers + self.assertEqualCollapsed(self.js, self.js_node.output()) def test_js_return_if_on(self): - output = u'<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' + output = '<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' self.assertEqual(output, self.js_node.output()) - def test_custom_output_dir(self): - try: - old_output_dir = settings.COMPRESS_OUTPUT_DIR - settings.COMPRESS_OUTPUT_DIR = 'custom' - output = u'<script type="text/javascript" src="/static/custom/js/066cd253eada.js"></script>' - self.assertEqual(output, JsCompressor(self.js).output()) - settings.COMPRESS_OUTPUT_DIR = '' - output = u'<script type="text/javascript" src="/static/js/066cd253eada.js"></script>' - self.assertEqual(output, JsCompressor(self.js).output()) - settings.COMPRESS_OUTPUT_DIR = '/custom/nested/' - output = u'<script type="text/javascript" src="/static/custom/nested/js/066cd253eada.js"></script>' - self.assertEqual(output, JsCompressor(self.js).output()) - finally: - settings.COMPRESS_OUTPUT_DIR = old_output_dir + @override_settings(COMPRESS_OUTPUT_DIR='custom') + def test_custom_output_dir1(self): + output = '<script type="text/javascript" src="/static/custom/js/066cd253eada.js"></script>' + self.assertEqual(output, JsCompressor(self.js).output()) + + @override_settings(COMPRESS_OUTPUT_DIR='') + def test_custom_output_dir2(self): + output = '<script type="text/javascript" src="/static/js/066cd253eada.js"></script>' + self.assertEqual(output, JsCompressor(self.js).output()) + + @override_settings(COMPRESS_OUTPUT_DIR='/custom/nested/') + def test_custom_output_dir3(self): + output = '<script type="text/javascript" src="/static/custom/nested/js/066cd253eada.js"></script>' + self.assertEqual(output, JsCompressor(self.js).output()) + @override_settings(COMPRESS_PRECOMPILERS=( + ('text/foobar', 'compressor.tests.test_base.TestPrecompiler'), + ), COMPRESS_ENABLED=True) def test_precompiler_class_used(self): - try: - original_precompilers = settings.COMPRESS_PRECOMPILERS - settings.COMPRESS_ENABLED = True - settings.COMPRESS_PRECOMPILERS = ( - ('text/foobar', 'compressor.tests.test_base.TestPrecompiler'), - ) - css = '<style type="text/foobar">p { border:10px solid red;}</style>' - css_node = CssCompressor(css) - output = BeautifulSoup(css_node.output('inline')) - self.assertEqual(output.text, 'OUTPUT') - finally: - settings.COMPRESS_PRECOMPILERS = original_precompilers + css = '<style type="text/foobar">p { border:10px solid red;}</style>' + css_node = CssCompressor(css) + output = make_soup(css_node.output('inline')) + self.assertEqual(output.text, 'OUTPUT') + @override_settings(COMPRESS_PRECOMPILERS=( + ('text/foobar', 'compressor.tests.test_base.NonexistentFilter'), + ), COMPRESS_ENABLED=True) def test_nonexistent_precompiler_class_error(self): - try: - original_precompilers = settings.COMPRESS_PRECOMPILERS - settings.COMPRESS_ENABLED = True - settings.COMPRESS_PRECOMPILERS = ( - ('text/foobar', 'compressor.tests.test_base.NonexistentFilter'), - ) - css = '<style type="text/foobar">p { border:10px solid red;}</style>' - css_node = CssCompressor(css) - self.assertRaises(FilterDoesNotExist, css_node.output, 'inline') - finally: - settings.COMPRESS_PRECOMPILERS = original_precompilers - - -class CssMediaTestCase(TestCase): + css = '<style type="text/foobar">p { border:10px solid red;}</style>' + css_node = CssCompressor(css) + self.assertRaises(FilterDoesNotExist, css_node.output, 'inline') + + +class CssMediaTestCase(SimpleTestCase): def setUp(self): self.css = """\ <link rel="stylesheet" href="/static/css/one.css" type="text/css" media="screen"> @@ -182,35 +219,41 @@ class CssMediaTestCase(TestCase): def test_css_output(self): css_node = CssCompressor(self.css) - links = BeautifulSoup(css_node.output()).findAll('link') - media = [u'screen', u'print', u'all', None] + if six.PY3: + links = make_soup(css_node.output()).find_all('link') + else: + links = make_soup(css_node.output()).findAll('link') + media = ['screen', 'print', 'all', None] self.assertEqual(len(links), 4) self.assertEqual(media, [l.get('media', None) for l in links]) def test_avoid_reordering_css(self): css = self.css + '<style type="text/css" media="print">p { border:10px solid red;}</style>' css_node = CssCompressor(css) - media = [u'screen', u'print', u'all', None, u'print'] - links = BeautifulSoup(css_node.output()).findAll('link') + media = ['screen', 'print', 'all', None, 'print'] + if six.PY3: + links = make_soup(css_node.output()).find_all('link') + else: + links = make_soup(css_node.output()).findAll('link') self.assertEqual(media, [l.get('media', None) for l in links]) + @override_settings(COMPRESS_PRECOMPILERS=( + ('text/foobar', 'python %s {infile} {outfile}' % os.path.join(test_dir, 'precompiler.py')), + ), COMPRESS_ENABLED=False) def test_passthough_when_compress_disabled(self): - original_precompilers = settings.COMPRESS_PRECOMPILERS - settings.COMPRESS_ENABLED = False - settings.COMPRESS_PRECOMPILERS = ( - ('text/foobar', 'python %s {infile} {outfile}' % os.path.join(test_dir, 'precompiler.py')), - ) css = """\ <link rel="stylesheet" href="/static/css/one.css" type="text/css" media="screen"> <link rel="stylesheet" href="/static/css/two.css" type="text/css" media="screen"> <style type="text/foobar" media="screen">h1 { border:5px solid green;}</style>""" css_node = CssCompressor(css) - output = BeautifulSoup(css_node.output()).findAll(['link', 'style']) - self.assertEqual([u'/static/css/one.css', u'/static/css/two.css', None], + if six.PY3: + output = make_soup(css_node.output()).find_all(['link', 'style']) + else: + output = make_soup(css_node.output()).findAll(['link', 'style']) + self.assertEqual(['/static/css/one.css', '/static/css/two.css', None], [l.get('href', None) for l in output]) - self.assertEqual([u'screen', u'screen', u'screen'], + self.assertEqual(['screen', 'screen', 'screen'], [l.get('media', None) for l in output]) - settings.COMPRESS_PRECOMPILERS = original_precompilers class VerboseTestCase(CompressorTestCase): diff --git a/compressor/tests/test_filters.py b/compressor/tests/test_filters.py index 90c4036..b656a65 100644 --- a/compressor/tests/test_filters.py +++ b/compressor/tests/test_filters.py @@ -1,9 +1,13 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals +import io import os import sys -from unittest2 import skipIf +import textwrap +from django.utils import six from django.test import TestCase +from django.utils import unittest +from django.test.utils import override_settings from compressor.cache import get_hashed_mtime, get_hashed_content from compressor.conf import settings @@ -16,56 +20,66 @@ from compressor.filters.template import TemplateFilter from compressor.tests.test_base import test_dir +@unittest.skipIf(find_command(settings.COMPRESS_CSSTIDY_BINARY) is None, + 'CSStidy binary %r not found' % settings.COMPRESS_CSSTIDY_BINARY) class CssTidyTestCase(TestCase): def test_tidy(self): - content = """ -/* Some comment */ -font,th,td,p{ -color: black; -} -""" + content = textwrap.dedent("""\ + /* Some comment */ + font,th,td,p{ + color: black; + } + """) from compressor.filters.csstidy import CSSTidyFilter + ret = CSSTidyFilter(content).input() + self.assertIsInstance(ret, six.text_type) self.assertEqual( "font,th,td,p{color:#000;}", CSSTidyFilter(content).input()) -CssTidyTestCase = skipIf( - find_command(settings.COMPRESS_CSSTIDY_BINARY) is None, - 'CSStidy binary %r not found' % settings.COMPRESS_CSSTIDY_BINARY, -)(CssTidyTestCase) - class PrecompilerTestCase(TestCase): - def setUp(self): self.filename = os.path.join(test_dir, 'static/css/one.css') - with open(self.filename) as f: - self.content = f.read() + with io.open(self.filename, encoding=settings.FILE_CHARSET) as file: + self.content = file.read() self.test_precompiler = os.path.join(test_dir, 'precompiler.py') def test_precompiler_infile_outfile(self): command = '%s %s -f {infile} -o {outfile}' % (sys.executable, self.test_precompiler) - compiler = CompilerFilter(content=self.content, filename=self.filename, command=command) - self.assertEqual(u"body { color:#990; }", compiler.input()) + compiler = CompilerFilter( + content=self.content, filename=self.filename, + charset=settings.FILE_CHARSET, command=command) + self.assertEqual("body { color:#990; }", compiler.input()) def test_precompiler_infile_stdout(self): command = '%s %s -f {infile}' % (sys.executable, self.test_precompiler) - compiler = CompilerFilter(content=self.content, filename=None, command=command) - self.assertEqual(u"body { color:#990; }%s" % os.linesep, compiler.input()) + compiler = CompilerFilter( + content=self.content, filename=None, charset=None, command=command) + self.assertEqual("body { color:#990; }%s" % os.linesep, compiler.input()) def test_precompiler_stdin_outfile(self): command = '%s %s -o {outfile}' % (sys.executable, self.test_precompiler) - compiler = CompilerFilter(content=self.content, filename=None, command=command) - self.assertEqual(u"body { color:#990; }", compiler.input()) + compiler = CompilerFilter( + content=self.content, filename=None, charset=None, command=command) + self.assertEqual("body { color:#990; }", compiler.input()) def test_precompiler_stdin_stdout(self): command = '%s %s' % (sys.executable, self.test_precompiler) - compiler = CompilerFilter(content=self.content, filename=None, command=command) - self.assertEqual(u"body { color:#990; }%s" % os.linesep, compiler.input()) + compiler = CompilerFilter( + content=self.content, filename=None, charset=None, command=command) + self.assertEqual("body { color:#990; }%s" % os.linesep, compiler.input()) def test_precompiler_stdin_stdout_filename(self): command = '%s %s' % (sys.executable, self.test_precompiler) + compiler = CompilerFilter( + content=self.content, filename=self.filename, + charset=settings.FILE_CHARSET, command=command) + self.assertEqual("body { color:#990; }%s" % os.linesep, compiler.input()) + + def test_precompiler_output_unicode(self): + command = '%s %s' % (sys.executable, self.test_precompiler) compiler = CompilerFilter(content=self.content, filename=self.filename, command=command) - self.assertEqual(u"body { color:#990; }%s" % os.linesep, compiler.input()) + self.assertEqual(type(compiler.input()), six.text_type) class CssMinTestCase(TestCase): @@ -77,7 +91,7 @@ class CssMinTestCase(TestCase): } -""" + """ output = "p{background:#369 url('../../images/image.gif')}" self.assertEqual(output, CSSMinFilter(content).output()) @@ -210,14 +224,14 @@ class CssAbsolutizingTestCase(TestCase): 'hash1': self.hashing_func(os.path.join(settings.COMPRESS_ROOT, 'img/python.png')), 'hash2': self.hashing_func(os.path.join(settings.COMPRESS_ROOT, 'img/add.png')), } - self.assertEqual([u"""\ + self.assertEqual(["""\ p { background: url('/static/img/python.png?%(hash1)s'); } p { background: url('/static/img/python.png?%(hash1)s'); } p { background: url('/static/img/python.png?%(hash1)s'); } p { background: url('/static/img/python.png?%(hash1)s'); } p { filter: progid:DXImageTransform.Microsoft.AlphaImageLoader(src='/static/img/python.png?%(hash1)s'); } """ % hash_dict, - u"""\ + """\ p { background: url('/static/img/add.png?%(hash2)s'); } p { background: url('/static/img/add.png?%(hash2)s'); } p { background: url('/static/img/add.png?%(hash2)s'); } @@ -264,7 +278,7 @@ class CssDataUriTestCase(TestCase): def test_data_uris(self): datauri_hash = get_hashed_mtime(os.path.join(settings.COMPRESS_ROOT, 'img/python.png')) - out = [u'''.add { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAJvSURBVDjLpZPrS5NhGIf9W7YvBYOkhlkoqCklWChv2WyKik7blnNris72bi6dus0DLZ0TDxW1odtopDs4D8MDZuLU0kXq61CijSIIasOvv94VTUfLiB74fXngup7nvrnvJABJ/5PfLnTTdcwOj4RsdYmo5glBWP6iOtzwvIKSWstI0Wgx80SBblpKtE9KQs/We7EaWoT/8wbWP61gMmCH0lMDvokT4j25TiQU/ITFkek9Ow6+7WH2gwsmahCPdwyw75uw9HEO2gUZSkfyI9zBPCJOoJ2SMmg46N61YO/rNoa39Xi41oFuXysMfh36/Fp0b7bAfWAH6RGi0HglWNCbzYgJaFjRv6zGuy+b9It96N3SQvNKiV9HvSaDfFEIxXItnPs23BzJQd6DDEVM0OKsoVwBG/1VMzpXVWhbkUM2K4oJBDYuGmbKIJ0qxsAbHfRLzbjcnUbFBIpx/qH3vQv9b3U03IQ/HfFkERTzfFj8w8jSpR7GBE123uFEYAzaDRIqX/2JAtJbDat/COkd7CNBva2cMvq0MGxp0PRSCPF8BXjWG3FgNHc9XPT71Ojy3sMFdfJRCeKxEsVtKwFHwALZfCUk3tIfNR8XiJwc1LmL4dg141JPKtj3WUdNFJqLGFVPC4OkR4BxajTWsChY64wmCnMxsWPCHcutKBxMVp5mxA1S+aMComToaqTRUQknLTH62kHOVEE+VQnjahscNCy0cMBWsSI0TCQcZc5ALkEYckL5A5noWSBhfm2AecMAjbcRWV0pUTh0HE64TNf0mczcnnQyu/MilaFJCae1nw2fbz1DnVOxyGTlKeZft/Ff8x1BRssfACjTwQAAAABJRU5ErkJggg=="); } + out = ['''.add { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAJvSURBVDjLpZPrS5NhGIf9W7YvBYOkhlkoqCklWChv2WyKik7blnNris72bi6dus0DLZ0TDxW1odtopDs4D8MDZuLU0kXq61CijSIIasOvv94VTUfLiB74fXngup7nvrnvJABJ/5PfLnTTdcwOj4RsdYmo5glBWP6iOtzwvIKSWstI0Wgx80SBblpKtE9KQs/We7EaWoT/8wbWP61gMmCH0lMDvokT4j25TiQU/ITFkek9Ow6+7WH2gwsmahCPdwyw75uw9HEO2gUZSkfyI9zBPCJOoJ2SMmg46N61YO/rNoa39Xi41oFuXysMfh36/Fp0b7bAfWAH6RGi0HglWNCbzYgJaFjRv6zGuy+b9It96N3SQvNKiV9HvSaDfFEIxXItnPs23BzJQd6DDEVM0OKsoVwBG/1VMzpXVWhbkUM2K4oJBDYuGmbKIJ0qxsAbHfRLzbjcnUbFBIpx/qH3vQv9b3U03IQ/HfFkERTzfFj8w8jSpR7GBE123uFEYAzaDRIqX/2JAtJbDat/COkd7CNBva2cMvq0MGxp0PRSCPF8BXjWG3FgNHc9XPT71Ojy3sMFdfJRCeKxEsVtKwFHwALZfCUk3tIfNR8XiJwc1LmL4dg141JPKtj3WUdNFJqLGFVPC4OkR4BxajTWsChY64wmCnMxsWPCHcutKBxMVp5mxA1S+aMComToaqTRUQknLTH62kHOVEE+VQnjahscNCy0cMBWsSI0TCQcZc5ALkEYckL5A5noWSBhfm2AecMAjbcRWV0pUTh0HE64TNf0mczcnnQyu/MilaFJCae1nw2fbz1DnVOxyGTlKeZft/Ff8x1BRssfACjTwQAAAABJRU5ErkJggg=="); } .add-with-hash { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAJvSURBVDjLpZPrS5NhGIf9W7YvBYOkhlkoqCklWChv2WyKik7blnNris72bi6dus0DLZ0TDxW1odtopDs4D8MDZuLU0kXq61CijSIIasOvv94VTUfLiB74fXngup7nvrnvJABJ/5PfLnTTdcwOj4RsdYmo5glBWP6iOtzwvIKSWstI0Wgx80SBblpKtE9KQs/We7EaWoT/8wbWP61gMmCH0lMDvokT4j25TiQU/ITFkek9Ow6+7WH2gwsmahCPdwyw75uw9HEO2gUZSkfyI9zBPCJOoJ2SMmg46N61YO/rNoa39Xi41oFuXysMfh36/Fp0b7bAfWAH6RGi0HglWNCbzYgJaFjRv6zGuy+b9It96N3SQvNKiV9HvSaDfFEIxXItnPs23BzJQd6DDEVM0OKsoVwBG/1VMzpXVWhbkUM2K4oJBDYuGmbKIJ0qxsAbHfRLzbjcnUbFBIpx/qH3vQv9b3U03IQ/HfFkERTzfFj8w8jSpR7GBE123uFEYAzaDRIqX/2JAtJbDat/COkd7CNBva2cMvq0MGxp0PRSCPF8BXjWG3FgNHc9XPT71Ojy3sMFdfJRCeKxEsVtKwFHwALZfCUk3tIfNR8XiJwc1LmL4dg141JPKtj3WUdNFJqLGFVPC4OkR4BxajTWsChY64wmCnMxsWPCHcutKBxMVp5mxA1S+aMComToaqTRUQknLTH62kHOVEE+VQnjahscNCy0cMBWsSI0TCQcZc5ALkEYckL5A5noWSBhfm2AecMAjbcRWV0pUTh0HE64TNf0mczcnnQyu/MilaFJCae1nw2fbz1DnVOxyGTlKeZft/Ff8x1BRssfACjTwQAAAABJRU5ErkJggg=="); } .python { background-image: url("/static/img/python.png?%s"); } .datauri { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9YGARc5KB0XV+IAAAAddEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIFRoZSBHSU1Q72QlbgAAAF1JREFUGNO9zL0NglAAxPEfdLTs4BZM4DIO4C7OwQg2JoQ9LE1exdlYvBBeZ7jqch9//q1uH4TLzw4d6+ErXMMcXuHWxId3KOETnnXXV6MJpcq2MLaI97CER3N0 vr4MkhoXe0rZigAAAABJRU5ErkJggg=="); } @@ -273,17 +287,11 @@ class CssDataUriTestCase(TestCase): class TemplateTestCase(TestCase): - def setUp(self): - self.old_context = settings.COMPRESS_TEMPLATE_FILTER_CONTEXT - - def tearDown(self): - settings.COMPRESS_TEMPLATE_FILTER_CONTEXT = self.old_context - + @override_settings(COMPRESS_TEMPLATE_FILTER_CONTEXT={ + 'stuff': 'thing', + 'gimmick': 'bold' + }) def test_template_filter(self): - settings.COMPRESS_TEMPLATE_FILTER_CONTEXT = { - 'stuff': 'thing', - 'gimmick': 'bold' - } content = """ #content {background-image: url("{{ STATIC_URL|default:stuff }}/images/bg.png");} #footer {font-weight: {{ gimmick }};} diff --git a/compressor/tests/test_jinja2ext.py b/compressor/tests/test_jinja2ext.py index cb2e012..5adc8ee 100644 --- a/compressor/tests/test_jinja2ext.py +++ b/compressor/tests/test_jinja2ext.py @@ -1,14 +1,18 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement +from __future__ import with_statement, unicode_literals -from django.test import TestCase +import sys -import jinja2 +from django.test import TestCase +from django.utils import unittest, six +from django.test.utils import override_settings from compressor.conf import settings from compressor.tests.test_base import css_tag +@unittest.skipUnless(not six.PY3 or sys.version_info[:2] >= (3, 3), + 'Jinja can only run on Python < 3 and >= 3.3') class TestJinja2CompressorExtension(TestCase): """ Test case for jinja2 extension. @@ -18,30 +22,34 @@ class TestJinja2CompressorExtension(TestCase): that we use jinja2 specific controls (*minus* character at block's beginning or end). For more information see jinja2 documentation. """ - def assertStrippedEqual(self, result, expected): self.assertEqual(result.strip(), expected.strip(), "%r != %r" % ( result.strip(), expected.strip())) def setUp(self): + import jinja2 + self.jinja2 = jinja2 from compressor.contrib.jinja2ext import CompressorExtension - self.env = jinja2.Environment(extensions=[CompressorExtension]) + self.env = self.jinja2.Environment(extensions=[CompressorExtension]) def test_error_raised_if_no_arguments_given(self): - self.assertRaises(jinja2.exceptions.TemplateSyntaxError, + self.assertRaises(self.jinja2.exceptions.TemplateSyntaxError, self.env.from_string, '{% compress %}Foobar{% endcompress %}') def test_error_raised_if_wrong_kind_given(self): - self.assertRaises(jinja2.exceptions.TemplateSyntaxError, + self.assertRaises(self.jinja2.exceptions.TemplateSyntaxError, self.env.from_string, '{% compress foo %}Foobar{% endcompress %}') + def test_error_raised_if_wrong_closing_kind_given(self): + self.assertRaises(self.jinja2.exceptions.TemplateSyntaxError, + self.env.from_string, '{% compress js %}Foobar{% endcompress css %}') + def test_error_raised_if_wrong_mode_given(self): - self.assertRaises(jinja2.exceptions.TemplateSyntaxError, + self.assertRaises(self.jinja2.exceptions.TemplateSyntaxError, self.env.from_string, '{% compress css foo %}Foobar{% endcompress %}') + @override_settings(COMPRESS_ENABLED=False) def test_compress_is_disabled(self): - org_COMPRESS_ENABLED = settings.COMPRESS_ENABLED - settings.COMPRESS_ENABLED = False tag_body = '\n'.join([ '<link rel="stylesheet" href="css/one.css" type="text/css" charset="utf-8">', '<style type="text/css">p { border:5px solid green;}</style>', @@ -50,16 +58,26 @@ class TestJinja2CompressorExtension(TestCase): template_string = '{% compress css %}' + tag_body + '{% endcompress %}' template = self.env.from_string(template_string) self.assertEqual(tag_body, template.render()) - settings.COMPRESS_ENABLED = org_COMPRESS_ENABLED + + # Test with explicit kind + template_string = '{% compress css %}' + tag_body + '{% endcompress css %}' + template = self.env.from_string(template_string) + self.assertEqual(tag_body, template.render()) def test_empty_tag(self): - template = self.env.from_string(u"""{% compress js %}{% block js %} + template = self.env.from_string("""{% compress js %}{% block js %} {% endblock %}{% endcompress %}""") context = {'STATIC_URL': settings.COMPRESS_URL} - self.assertEqual(u'', template.render(context)) + self.assertEqual('', template.render(context)) + + def test_empty_tag_with_kind(self): + template = self.env.from_string("""{% compress js %}{% block js %} + {% endblock %}{% endcompress js %}""") + context = {'STATIC_URL': settings.COMPRESS_URL} + self.assertEqual('', template.render(context)) def test_css_tag(self): - template = self.env.from_string(u"""{% compress css -%} + template = self.env.from_string("""{% compress css -%} <link rel="stylesheet" href="{{ STATIC_URL }}css/one.css" type="text/css" charset="utf-8"> <style type="text/css">p { border:5px solid green;}</style> <link rel="stylesheet" href="{{ STATIC_URL }}css/two.css" type="text/css" charset="utf-8"> @@ -69,7 +87,7 @@ class TestJinja2CompressorExtension(TestCase): self.assertEqual(out, template.render(context)) def test_nonascii_css_tag(self): - template = self.env.from_string(u"""{% compress css -%} + template = self.env.from_string("""{% compress css -%} <link rel="stylesheet" href="{{ STATIC_URL }}css/nonasc.css" type="text/css" charset="utf-8"> <style type="text/css">p { border:5px solid green;}</style> {% endcompress %}""") @@ -78,34 +96,34 @@ class TestJinja2CompressorExtension(TestCase): self.assertEqual(out, template.render(context)) def test_js_tag(self): - template = self.env.from_string(u"""{% compress js -%} + template = self.env.from_string("""{% compress js -%} <script src="{{ STATIC_URL }}js/one.js" type="text/javascript" charset="utf-8"></script> <script type="text/javascript" charset="utf-8">obj.value = "value";</script> {% endcompress %}""") context = {'STATIC_URL': settings.COMPRESS_URL} - out = u'<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' + out = '<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' self.assertEqual(out, template.render(context)) def test_nonascii_js_tag(self): - template = self.env.from_string(u"""{% compress js -%} + template = self.env.from_string("""{% compress js -%} <script src="{{ STATIC_URL }}js/nonasc.js" type="text/javascript" charset="utf-8"></script> <script type="text/javascript" charset="utf-8">var test_value = "\u2014";</script> {% endcompress %}""") context = {'STATIC_URL': settings.COMPRESS_URL} - out = u'<script type="text/javascript" src="/static/CACHE/js/e214fe629b28.js"></script>' + out = '<script type="text/javascript" src="/static/CACHE/js/e214fe629b28.js"></script>' self.assertEqual(out, template.render(context)) def test_nonascii_latin1_js_tag(self): - template = self.env.from_string(u"""{% compress js -%} + template = self.env.from_string("""{% compress js -%} <script src="{{ STATIC_URL }}js/nonasc-latin1.js" type="text/javascript" charset="latin-1"></script> <script type="text/javascript">var test_value = "\u2014";</script> {% endcompress %}""") context = {'STATIC_URL': settings.COMPRESS_URL} - out = u'<script type="text/javascript" src="/static/CACHE/js/be9e078b5ca7.js"></script>' + out = '<script type="text/javascript" src="/static/CACHE/js/be9e078b5ca7.js"></script>' self.assertEqual(out, template.render(context)) def test_css_inline(self): - template = self.env.from_string(u"""{% compress css, inline -%} + template = self.env.from_string("""{% compress css, inline -%} <link rel="stylesheet" href="{{ STATIC_URL }}css/one.css" type="text/css" charset="utf-8"> <style type="text/css">p { border:5px solid green;}</style> {% endcompress %}""") @@ -117,7 +135,7 @@ class TestJinja2CompressorExtension(TestCase): self.assertEqual(out, template.render(context)) def test_js_inline(self): - template = self.env.from_string(u"""{% compress js, inline -%} + template = self.env.from_string("""{% compress js, inline -%} <script src="{{ STATIC_URL }}js/one.js" type="text/css" type="text/javascript" charset="utf-8"></script> <script type="text/javascript" charset="utf-8">obj.value = "value";</script> {% endcompress %}""") @@ -128,11 +146,11 @@ class TestJinja2CompressorExtension(TestCase): def test_nonascii_inline_css(self): org_COMPRESS_ENABLED = settings.COMPRESS_ENABLED settings.COMPRESS_ENABLED = False - template = self.env.from_string(u'{% compress css %}' - u'<style type="text/css">' - u'/* русский текст */' - u'</style>{% endcompress %}') - out = u'<link rel="stylesheet" href="/static/CACHE/css/b2cec0f8cb24.css" type="text/css" />' + template = self.env.from_string('{% compress css %}' + '<style type="text/css">' + '/* русский текст */' + '</style>{% endcompress %}') + out = '<link rel="stylesheet" href="/static/CACHE/css/b2cec0f8cb24.css" type="text/css" />' settings.COMPRESS_ENABLED = org_COMPRESS_ENABLED context = {'STATIC_URL': settings.COMPRESS_URL} self.assertEqual(out, template.render(context)) diff --git a/compressor/tests/test_offline.py b/compressor/tests/test_offline.py index a988afd..327b901 100644 --- a/compressor/tests/test_offline.py +++ b/compressor/tests/test_offline.py @@ -1,12 +1,12 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals +import io import os -from StringIO import StringIO -from unittest2 import skipIf +import sys -import django +from django.core.management.base import CommandError from django.template import Template, Context from django.test import TestCase -from django.core.management.base import CommandError +from django.utils import six, unittest from compressor.cache import flush_offline_manifest, get_offline_manifest from compressor.conf import settings @@ -14,6 +14,22 @@ from compressor.exceptions import OfflineGenerationError from compressor.management.commands.compress import Command as CompressCommand from compressor.storage import default_storage +if six.PY3: + # there is an 'io' module in python 2.6+, but io.StringIO does not + # accept regular strings, just unicode objects + from io import StringIO +else: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + +# The Jinja2 tests fail on Python 3.2 due to the following: +# The line in compressor/management/commands/compress.py: +# compressor_nodes.setdefault(template, []).extend(nodes) +# causes the error "unhashable type: 'Template'" +_TEST_JINJA2 = not(sys.version_info[0] == 3 and sys.version_info[1] == 2) + class OfflineTestCaseMixin(object): template_name = "test_compressor_offline.html" @@ -21,6 +37,11 @@ class OfflineTestCaseMixin(object): # Change this for each test class templates_dir = "" expected_hash = "" + # Engines to test + if _TEST_JINJA2: + engines = ("django", "jinja2") + else: + engines = ("django",) def setUp(self): self._old_compress = settings.COMPRESS_ENABLED @@ -32,48 +53,128 @@ class OfflineTestCaseMixin(object): # Reset template dirs, because it enables us to force compress to # consider only a specific directory (helps us make true, # independant unit tests). - settings.TEMPLATE_DIRS = ( - os.path.join(settings.TEST_DIR, 'test_templates', self.templates_dir), - ) + # Specify both Jinja2 and Django template locations. When the wrong engine + # is used to parse a template, the TemplateSyntaxError will cause the + # template to be skipped over. + django_template_dir = os.path.join(settings.TEST_DIR, 'test_templates', self.templates_dir) + jinja2_template_dir = os.path.join(settings.TEST_DIR, 'test_templates_jinja2', self.templates_dir) + settings.TEMPLATE_DIRS = (django_template_dir, jinja2_template_dir) + # Enable offline compress settings.COMPRESS_ENABLED = True settings.COMPRESS_OFFLINE = True - self.template_path = os.path.join(settings.TEMPLATE_DIRS[0], self.template_name) - self.template_file = open(self.template_path) - self.template = Template(self.template_file.read().decode(settings.FILE_CHARSET)) + + if "django" in self.engines: + self.template_path = os.path.join(django_template_dir, self.template_name) + + with io.open(self.template_path, encoding=settings.FILE_CHARSET) as file: + self.template = Template(file.read()) + + self._old_jinja2_get_environment = settings.COMPRESS_JINJA2_GET_ENVIRONMENT + + if "jinja2" in self.engines: + # Setup Jinja2 settings. + settings.COMPRESS_JINJA2_GET_ENVIRONMENT = lambda: self._get_jinja2_env() + jinja2_env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT() + self.template_path_jinja2 = os.path.join(jinja2_template_dir, self.template_name) + + with io.open(self.template_path_jinja2, encoding=settings.FILE_CHARSET) as file: + self.template_jinja2 = jinja2_env.from_string(file.read()) def tearDown(self): + settings.COMPRESS_JINJA2_GET_ENVIRONMENT = self._old_jinja2_get_environment settings.COMPRESS_ENABLED = self._old_compress settings.COMPRESS_OFFLINE = self._old_compress_offline settings.TEMPLATE_DIRS = self._old_template_dirs - self.template_file.close() manifest_path = os.path.join('CACHE', 'manifest.json') if default_storage.exists(manifest_path): default_storage.delete(manifest_path) - def test_offline(self): - count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity) + def _render_template(self, engine): + if engine == "django": + return self.template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) + elif engine == "jinja2": + return self.template_jinja2.render(settings.COMPRESS_OFFLINE_CONTEXT) + "\n" + else: + return None + + def _test_offline(self, engine): + count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity, engine=engine) self.assertEqual(1, count) self.assertEqual([ - u'<script type="text/javascript" src="/static/CACHE/js/%s.js"></script>' % (self.expected_hash, ), + '<script type="text/javascript" src="/static/CACHE/js/%s.js"></script>' % (self.expected_hash, ), ], result) - rendered_template = self.template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) + rendered_template = self._render_template(engine) self.assertEqual(rendered_template, "".join(result) + "\n") + def test_offline(self): + for engine in self.engines: + self._test_offline(engine=engine) + + def _get_jinja2_env(self): + import jinja2 + import jinja2.ext + from compressor.offline.jinja2 import url_for, SpacelessExtension + from compressor.contrib.jinja2ext import CompressorExtension + + # Extensions needed for the test cases only. + extensions = [ + CompressorExtension, + SpacelessExtension, + jinja2.ext.with_, + jinja2.ext.do, + ] + loader = self._get_jinja2_loader() + env = jinja2.Environment(extensions=extensions, loader=loader) + env.globals['url_for'] = url_for + + return env + + def _get_jinja2_loader(self): + import jinja2 + + loader = jinja2.FileSystemLoader(settings.TEMPLATE_DIRS, encoding=settings.FILE_CHARSET) + return loader + + +class OfflineGenerationSkipDuplicatesTestCase(OfflineTestCaseMixin, TestCase): + templates_dir = "test_duplicate" + + # We don't need to test multiples engines here. + engines = ("django",) + + def _test_offline(self, engine): + count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity, engine=engine) + # Only one block compressed, the second identical one was skipped. + self.assertEqual(1, count) + # Only 1 <script> block in returned result as well. + self.assertEqual([ + '<script type="text/javascript" src="/static/CACHE/js/f5e179b8eca4.js"></script>', + ], result) + rendered_template = self._render_template(engine) + # But rendering the template returns both (identical) scripts. + self.assertEqual(rendered_template, "".join(result * 2) + "\n") + class OfflineGenerationBlockSuperTestCase(OfflineTestCaseMixin, TestCase): templates_dir = "test_block_super" expected_hash = "7c02d201f69d" + # Block.super not supported for Jinja2 yet. + engines = ("django",) class OfflineGenerationBlockSuperMultipleTestCase(OfflineTestCaseMixin, TestCase): templates_dir = "test_block_super_multiple" - expected_hash = "2f6ef61c488e" + expected_hash = "f8891c416981" + # Block.super not supported for Jinja2 yet. + engines = ("django",) class OfflineGenerationBlockSuperMultipleWithCachedLoaderTestCase(OfflineTestCaseMixin, TestCase): templates_dir = "test_block_super_multiple_cached" expected_hash = "2f6ef61c488e" + # Block.super not supported for Jinja2 yet. + engines = ("django",) def setUp(self): self._old_template_loaders = settings.TEMPLATE_LOADERS @@ -92,15 +193,17 @@ class OfflineGenerationBlockSuperMultipleWithCachedLoaderTestCase(OfflineTestCas class OfflineGenerationBlockSuperTestCaseWithExtraContent(OfflineTestCaseMixin, TestCase): templates_dir = "test_block_super_extra" + # Block.super not supported for Jinja2 yet. + engines = ("django",) - def test_offline(self): - count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity) + def _test_offline(self, engine): + count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity, engine=engine) self.assertEqual(2, count) self.assertEqual([ - u'<script type="text/javascript" src="/static/CACHE/js/ced14aec5856.js"></script>', - u'<script type="text/javascript" src="/static/CACHE/js/7c02d201f69d.js"></script>' + '<script type="text/javascript" src="/static/CACHE/js/ced14aec5856.js"></script>', + '<script type="text/javascript" src="/static/CACHE/js/7c02d201f69d.js"></script>' ], result) - rendered_template = self.template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) + rendered_template = self._render_template(engine) self.assertEqual(rendered_template, "".join(result) + "\n") @@ -128,10 +231,6 @@ class OfflineGenerationTemplateTagTestCase(OfflineTestCaseMixin, TestCase): class OfflineGenerationStaticTemplateTagTestCase(OfflineTestCaseMixin, TestCase): templates_dir = "test_static_templatetag" expected_hash = "dfa2bb387fa8" -# This test uses {% static %} which was introduced in django 1.4 -OfflineGenerationStaticTemplateTagTestCase = skipIf( - django.VERSION[1] < 4, 'Django 1.4 not found' -)(OfflineGenerationStaticTemplateTagTestCase) class OfflineGenerationTestCaseWithContext(OfflineTestCaseMixin, TestCase): @@ -153,11 +252,23 @@ class OfflineGenerationTestCaseWithContext(OfflineTestCaseMixin, TestCase): class OfflineGenerationTestCaseErrors(OfflineTestCaseMixin, TestCase): templates_dir = "test_error_handling" - def test_offline(self): - count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity) - self.assertEqual(2, count) - self.assertIn(u'<script type="text/javascript" src="/static/CACHE/js/3872c9ae3f42.js"></script>', result) - self.assertIn(u'<script type="text/javascript" src="/static/CACHE/js/cd8870829421.js"></script>', result) + def _test_offline(self, engine): + count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity, engine=engine) + + if engine == "django": + self.assertEqual(2, count) + else: + # Because we use env.parse in Jinja2Parser, the engine does not + # actually load the "extends" and "includes" templates, and so + # it is unable to detect that they are missing. So all the "compress" + # nodes are processed correctly. + self.assertEqual(4, count) + self.assertEqual(engine, "jinja2") + self.assertIn('<link rel="stylesheet" href="/static/CACHE/css/78bd7a762e2d.css" type="text/css" />', result) + self.assertIn('<link rel="stylesheet" href="/static/CACHE/css/e31030430724.css" type="text/css" />', result) + + self.assertIn('<script type="text/javascript" src="/static/CACHE/js/3872c9ae3f42.js"></script>', result) + self.assertIn('<script type="text/javascript" src="/static/CACHE/js/cd8870829421.js"></script>', result) class OfflineGenerationTestCaseWithError(OfflineTestCaseMixin, TestCase): @@ -168,7 +279,7 @@ class OfflineGenerationTestCaseWithError(OfflineTestCaseMixin, TestCase): settings.COMPRESS_PRECOMPILERS = (('text/coffeescript', 'non-existing-binary'),) super(OfflineGenerationTestCaseWithError, self).setUp() - def test_offline(self): + def _test_offline(self, engine): """ Test that a CommandError is raised with DEBUG being False as well as True, as otherwise errors in configuration will never show in @@ -178,10 +289,10 @@ class OfflineGenerationTestCaseWithError(OfflineTestCaseMixin, TestCase): try: settings.DEBUG = True - self.assertRaises(CommandError, CompressCommand().compress) + self.assertRaises(CommandError, CompressCommand().compress, engine=engine) settings.DEBUG = False - self.assertRaises(CommandError, CompressCommand().compress) + self.assertRaises(CommandError, CompressCommand().compress, engine=engine) finally: settings.DEBUG = self._old_debug @@ -201,19 +312,30 @@ class OfflineGenerationTestCase(OfflineTestCaseMixin, TestCase): self.assertRaises(OfflineGenerationError, self.template.render, Context({})) - def test_deleting_manifest_does_not_affect_rendering(self): - count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity) + @unittest.skipIf(not _TEST_JINJA2, "No Jinja2 testing") + def test_rendering_without_manifest_raises_exception_jinja2(self): + # flush cached manifest + flush_offline_manifest() + self.assertRaises(OfflineGenerationError, + self.template_jinja2.render, {}) + + def _test_deleting_manifest_does_not_affect_rendering(self, engine): + count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity, engine=engine) get_offline_manifest() manifest_path = os.path.join('CACHE', 'manifest.json') if default_storage.exists(manifest_path): default_storage.delete(manifest_path) self.assertEqual(1, count) self.assertEqual([ - u'<script type="text/javascript" src="/static/CACHE/js/%s.js"></script>' % (self.expected_hash, ), + '<script type="text/javascript" src="/static/CACHE/js/%s.js"></script>' % (self.expected_hash, ), ], result) - rendered_template = self.template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) + rendered_template = self._render_template(engine) self.assertEqual(rendered_template, "".join(result) + "\n") + def test_deleting_manifest_does_not_affect_rendering(self): + for engine in self.engines: + self._test_deleting_manifest_does_not_affect_rendering(engine) + def test_requires_model_validation(self): self.assertFalse(CompressCommand.requires_model_validation) @@ -238,13 +360,50 @@ class OfflineGenerationTestCase(OfflineTestCaseMixin, TestCase): settings.TEMPLATE_LOADERS = old_loaders +class OfflineGenerationBlockSuperBaseCompressed(OfflineTestCaseMixin, TestCase): + template_names = ["base.html", "base2.html", "test_compressor_offline.html"] + templates_dir = 'test_block_super_base_compressed' + expected_hash = ['028c3fc42232', '2e9d3f5545a6', 'f8891c416981'] + # Block.super not supported for Jinja2 yet. + engines = ("django",) + + def setUp(self): + super(OfflineGenerationBlockSuperBaseCompressed, self).setUp() + + self.template_paths = [] + self.templates = [] + for template_name in self.template_names: + template_path = os.path.join(settings.TEMPLATE_DIRS[0], template_name) + self.template_paths.append(template_path) + with io.open(template_path, encoding=settings.FILE_CHARSET) as file: + template = Template(file.read()) + self.templates.append(template) + + def _render_template(self, template, engine): + if engine == "django": + return template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) + elif engine == "jinja2": + return template.render(settings.COMPRESS_OFFLINE_CONTEXT) + "\n" + else: + return None + + def _test_offline(self, engine): + count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity, engine=engine) + self.assertEqual(len(self.expected_hash), count) + for expected_hash, template in zip(self.expected_hash, self.templates): + expected_output = '<script type="text/javascript" src="/static/CACHE/js/%s.js"></script>' % (expected_hash, ) + self.assertIn(expected_output, result) + rendered_template = self._render_template(template, engine) + self.assertEqual(rendered_template, expected_output + '\n') + + class OfflineGenerationInlineNonAsciiTestCase(OfflineTestCaseMixin, TestCase): templates_dir = "test_inline_non_ascii" def setUp(self): self.old_offline_context = settings.COMPRESS_OFFLINE_CONTEXT settings.COMPRESS_OFFLINE_CONTEXT = { - 'test_non_ascii_value': u'\u2014', + 'test_non_ascii_value': '\u2014', } super(OfflineGenerationInlineNonAsciiTestCase, self).setUp() @@ -252,7 +411,89 @@ class OfflineGenerationInlineNonAsciiTestCase(OfflineTestCaseMixin, TestCase): self.COMPRESS_OFFLINE_CONTEXT = self.old_offline_context super(OfflineGenerationInlineNonAsciiTestCase, self).tearDown() - def test_offline(self): - count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity) - rendered_template = self.template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) + def _test_offline(self, engine): + count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity, engine=engine) + rendered_template = self._render_template(engine) self.assertEqual(rendered_template, "".join(result) + "\n") + + +class OfflineGenerationComplexTestCase(OfflineTestCaseMixin, TestCase): + templates_dir = "test_complex" + + def setUp(self): + self.old_offline_context = settings.COMPRESS_OFFLINE_CONTEXT + settings.COMPRESS_OFFLINE_CONTEXT = { + 'condition': 'OK!', + # Django templating does not allow definition of tuples in the + # templates. Make sure this is same as test_templates_jinja2/test_complex. + 'my_names': ("js/one.js", "js/nonasc.js"), + } + super(OfflineGenerationComplexTestCase, self).setUp() + + def tearDown(self): + self.COMPRESS_OFFLINE_CONTEXT = self.old_offline_context + super(OfflineGenerationComplexTestCase, self).tearDown() + + def _test_offline(self, engine): + count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity, engine=engine) + self.assertEqual(3, count) + self.assertEqual([ + '<script type="text/javascript" src="/static/CACHE/js/0e8807bebcee.js"></script>', + '<script type="text/javascript" src="/static/CACHE/js/eed1d222933e.js"></script>', + '<script type="text/javascript" src="/static/CACHE/js/00b4baffe335.js"></script>', + ], result) + rendered_template = self._render_template(engine) + result = (result[0], result[2]) + self.assertEqual(rendered_template, "".join(result) + "\n") + + +# Coffin does not work on Python 3.2+ due to: +# The line at coffin/template/__init__.py:15 +# from library import * +# causing 'ImportError: No module named library'. +# It seems there is no evidence nor indicated support for Python 3+. +@unittest.skipIf(sys.version_info >= (3, 2), + "Coffin does not support 3.2+") +class OfflineGenerationCoffinTestCase(OfflineTestCaseMixin, TestCase): + templates_dir = "test_coffin" + expected_hash = "32c8281e3346" + engines = ("jinja2",) + + def _get_jinja2_env(self): + import jinja2 + from coffin.common import env + from compressor.contrib.jinja2ext import CompressorExtension + + # Could have used the env.add_extension method, but it's only available + # in Jinja2 v2.5 + new_env = jinja2.Environment(extensions=[CompressorExtension]) + env.extensions.update(new_env.extensions) + + return env + + +# Jingo does not work when using Python 3.2 due to the use of Unicode string +# prefix (and possibly other stuff), but it actually works when using Python 3.3 +# since it tolerates the use of the Unicode string prefix. Python 3.3 support +# is also evident in its tox.ini file. +@unittest.skipIf(sys.version_info >= (3, 2) and sys.version_info < (3, 3), + "Jingo does not support 3.2") +class OfflineGenerationJingoTestCase(OfflineTestCaseMixin, TestCase): + templates_dir = "test_jingo" + expected_hash = "61ec584468eb" + engines = ("jinja2",) + + def _get_jinja2_env(self): + import jinja2 + import jinja2.ext + from jingo import env + from compressor.contrib.jinja2ext import CompressorExtension + from compressor.offline.jinja2 import SpacelessExtension, url_for + + # Could have used the env.add_extension method, but it's only available + # in Jinja2 v2.5 + new_env = jinja2.Environment(extensions=[CompressorExtension, SpacelessExtension, jinja2.ext.with_]) + env.extensions.update(new_env.extensions) + env.globals['url_for'] = url_for + + return env diff --git a/compressor/tests/test_parsers.py b/compressor/tests/test_parsers.py index 04ec924..d9b4dd6 100644 --- a/compressor/tests/test_parsers.py +++ b/compressor/tests/test_parsers.py @@ -1,6 +1,5 @@ from __future__ import with_statement import os -from unittest2 import skipIf try: import lxml @@ -17,15 +16,15 @@ try: except ImportError: BeautifulSoup = None +from django.utils import unittest +from django.test.utils import override_settings from compressor.base import SOURCE_HUNK, SOURCE_FILE from compressor.conf import settings -from compressor.css import CssCompressor from compressor.tests.test_base import CompressorTestCase class ParserTestCase(object): - def setUp(self): self.old_parser = settings.COMPRESS_PARSER settings.COMPRESS_PARSER = self.parser_cls @@ -35,52 +34,92 @@ class ParserTestCase(object): settings.COMPRESS_PARSER = self.old_parser +@unittest.skipIf(lxml is None, 'lxml not found') class LxmlParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.LxmlParser' -LxmlParserTests = skipIf(lxml is None, 'lxml not found')(LxmlParserTests) +@unittest.skipIf(html5lib is None, 'html5lib not found') class Html5LibParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.Html5LibParser' - - def setUp(self): - super(Html5LibParserTests, self).setUp() - # special version of the css since the parser sucks - self.css = """\ -<link href="/static/css/one.css" rel="stylesheet" type="text/css"> -<style type="text/css">p { border:5px solid green;}</style> -<link href="/static/css/two.css" rel="stylesheet" type="text/css">""" - self.css_node = CssCompressor(self.css) + # Special test variants required since xml.etree holds attributes + # as a plain dictionary, e.g. key order is unpredictable. def test_css_split(self): - out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'one.css'), u'css/one.css', u'<link href="/static/css/one.css" rel="stylesheet" type="text/css">'), - (SOURCE_HUNK, u'p { border:5px solid green;}', None, u'<style type="text/css">p { border:5px solid green;}</style>'), - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'two.css'), u'css/two.css', u'<link href="/static/css/two.css" rel="stylesheet" type="text/css">'), - ] split = self.css_node.split_contents() - split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split] - self.assertEqual(out, split) + out0 = ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'css', 'one.css'), + 'css/one.css', + '{http://www.w3.org/1999/xhtml}link', + {'rel': 'stylesheet', 'href': '/static/css/one.css', + 'type': 'text/css'}, + ) + self.assertEqual(out0, split[0][:3] + (split[0][3].tag, + split[0][3].attrib)) + out1 = ( + SOURCE_HUNK, + 'p { border:5px solid green;}', + None, + '<style type="text/css">p { border:5px solid green;}</style>', + ) + self.assertEqual(out1, split[1][:3] + + (self.css_node.parser.elem_str(split[1][3]),)) + out2 = ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'css', 'two.css'), + 'css/two.css', + '{http://www.w3.org/1999/xhtml}link', + {'rel': 'stylesheet', 'href': '/static/css/two.css', + 'type': 'text/css'}, + ) + self.assertEqual(out2, split[2][:3] + (split[2][3].tag, + split[2][3].attrib)) def test_js_split(self): - out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'js', u'one.js'), u'js/one.js', u'<script src="/static/js/one.js" type="text/javascript"></script>'), - (SOURCE_HUNK, u'obj.value = "value";', None, u'<script type="text/javascript">obj.value = "value";</script>'), - ] split = self.js_node.split_contents() - split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split] - self.assertEqual(out, split) - -Html5LibParserTests = skipIf( - html5lib is None, 'html5lib not found')(Html5LibParserTests) - - + out0 = ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'js', 'one.js'), + 'js/one.js', + '{http://www.w3.org/1999/xhtml}script', + {'src': '/static/js/one.js', 'type': 'text/javascript'}, + None, + ) + self.assertEqual(out0, split[0][:3] + (split[0][3].tag, + split[0][3].attrib, + split[0][3].text)) + out1 = ( + SOURCE_HUNK, + 'obj.value = "value";', + None, + '{http://www.w3.org/1999/xhtml}script', + {'type': 'text/javascript'}, + 'obj.value = "value";', + ) + self.assertEqual(out1, split[1][:3] + (split[1][3].tag, + split[1][3].attrib, + split[1][3].text)) + + def test_css_return_if_off(self): + settings.COMPRESS_ENABLED = False + # Yes, they are semantically equal but attributes might be + # scrambled in unpredictable order. A more elaborate check + # would require parsing both arguments with a different parser + # and then evaluating the result, which no longer is + # a meaningful unit test. + self.assertEqual(len(self.css), len(self.css_node.output())) + + @override_settings(COMPRESS_PRECOMPILERS=(), COMPRESS_ENABLED=False) + def test_js_return_if_off(self): + # As above. + self.assertEqual(len(self.js), len(self.js_node.output())) + + +@unittest.skipIf(BeautifulSoup is None, 'BeautifulSoup not found') class BeautifulSoupParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.BeautifulSoupParser' -BeautifulSoupParserTests = skipIf( - BeautifulSoup is None, 'BeautifulSoup not found')(BeautifulSoupParserTests) - class HtmlParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.HtmlParser' diff --git a/compressor/tests/test_signals.py b/compressor/tests/test_signals.py index b4ece9c..13d5eed 100644 --- a/compressor/tests/test_signals.py +++ b/compressor/tests/test_signals.py @@ -11,7 +11,7 @@ from compressor.signals import post_compress class PostCompressSignalTestCase(TestCase): def setUp(self): settings.COMPRESS_ENABLED = True - settings.COMPRESS_PRECOMPILERS = {} + settings.COMPRESS_PRECOMPILERS = () settings.COMPRESS_DEBUG_TOGGLE = 'nocompress' self.css = """\ <link rel="stylesheet" href="/static/css/one.css" type="text/css" /> @@ -34,9 +34,9 @@ class PostCompressSignalTestCase(TestCase): post_compress.connect(callback) self.js_node.output() args, kwargs = callback.call_args - self.assertEquals(JsCompressor, kwargs['sender']) - self.assertEquals('js', kwargs['type']) - self.assertEquals('file', kwargs['mode']) + self.assertEqual(JsCompressor, kwargs['sender']) + self.assertEqual('js', kwargs['type']) + self.assertEqual('file', kwargs['mode']) context = kwargs['context'] assert 'url' in context['compressed'] @@ -47,9 +47,9 @@ class PostCompressSignalTestCase(TestCase): post_compress.connect(callback) self.css_node.output() args, kwargs = callback.call_args - self.assertEquals(CssCompressor, kwargs['sender']) - self.assertEquals('css', kwargs['type']) - self.assertEquals('file', kwargs['mode']) + self.assertEqual(CssCompressor, kwargs['sender']) + self.assertEqual('css', kwargs['type']) + self.assertEqual('file', kwargs['mode']) context = kwargs['context'] assert 'url' in context['compressed'] @@ -65,4 +65,4 @@ class PostCompressSignalTestCase(TestCase): callback = Mock(wraps=listener) post_compress.connect(callback) css_node.output() - self.assertEquals(3, callback.call_count) + self.assertEqual(3, callback.call_count) diff --git a/compressor/tests/test_storages.py b/compressor/tests/test_storages.py index 713002e..91a36f2 100644 --- a/compressor/tests/test_storages.py +++ b/compressor/tests/test_storages.py @@ -1,29 +1,41 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals import errno import os from django.core.files.base import ContentFile from django.core.files.storage import get_storage_class from django.test import TestCase +from django.utils.functional import LazyObject -from compressor import base +from compressor import storage from compressor.conf import settings from compressor.tests.test_base import css_tag from compressor.tests.test_templatetags import render +class GzipStorage(LazyObject): + def _setup(self): + self._wrapped = get_storage_class('compressor.storage.GzipCompressorFileStorage')() + + class StorageTestCase(TestCase): def setUp(self): - self._storage = base.default_storage - base.default_storage = get_storage_class( - 'compressor.storage.GzipCompressorFileStorage')() + self.old_enabled = settings.COMPRESS_ENABLED settings.COMPRESS_ENABLED = True + self.default_storage = storage.default_storage + storage.default_storage = GzipStorage() def tearDown(self): - base.default_storage = self._storage + storage.default_storage = self.default_storage + settings.COMPRESS_ENABLED = self.old_enabled + + def test_gzip_storage(self): + storage.default_storage.save('test.txt', ContentFile('yeah yeah')) + self.assertTrue(os.path.exists(os.path.join(settings.COMPRESS_ROOT, 'test.txt'))) + self.assertTrue(os.path.exists(os.path.join(settings.COMPRESS_ROOT, 'test.txt.gz'))) def test_css_tag_with_storage(self): - template = u"""{% load compress %}{% compress css %} + template = """{% load compress %}{% compress css %} <link rel="stylesheet" href="{{ STATIC_URL }}css/one.css" type="text/css"> <style type="text/css">p { border:5px solid white;}</style> <link rel="stylesheet" href="{{ STATIC_URL }}css/two.css" type="text/css"> @@ -40,13 +52,13 @@ class StorageTestCase(TestCase): def race_remove(path): "Patched os.remove to raise ENOENT (No such file or directory)" original_remove(path) - raise OSError(errno.ENOENT, u'Fake ENOENT') + raise OSError(errno.ENOENT, 'Fake ENOENT') try: os.remove = race_remove - self._storage.save('race.file', ContentFile('Fake ENOENT')) - self._storage.delete('race.file') - self.assertFalse(self._storage.exists('race.file')) + self.default_storage.save('race.file', ContentFile('Fake ENOENT')) + self.default_storage.delete('race.file') + self.assertFalse(self.default_storage.exists('race.file')) finally: # Restore os.remove os.remove = original_remove diff --git a/compressor/tests/test_templates/test_block_super_base_compressed/base.html b/compressor/tests/test_templates/test_block_super_base_compressed/base.html new file mode 100644 index 0000000..481ff40 --- /dev/null +++ b/compressor/tests/test_templates/test_block_super_base_compressed/base.html @@ -0,0 +1,10 @@ +{% load compress %}{% spaceless %} + +{% compress js %} +{% block js %} + <script type="text/javascript"> + alert("test using multiple inheritance and block.super"); + </script> +{% endblock %} +{% endcompress %} +{% endspaceless %} diff --git a/compressor/tests/test_templates/test_block_super_base_compressed/base2.html b/compressor/tests/test_templates/test_block_super_base_compressed/base2.html new file mode 100644 index 0000000..abd074d --- /dev/null +++ b/compressor/tests/test_templates/test_block_super_base_compressed/base2.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block js %}{% spaceless %} + {{ block.super }} + <script type="text/javascript"> + alert("this alert should be included"); + </script> +{% endspaceless %}{% endblock %} diff --git a/compressor/tests/test_templates/test_block_super_base_compressed/test_compressor_offline.html b/compressor/tests/test_templates/test_block_super_base_compressed/test_compressor_offline.html new file mode 100644 index 0000000..01382ec --- /dev/null +++ b/compressor/tests/test_templates/test_block_super_base_compressed/test_compressor_offline.html @@ -0,0 +1,8 @@ +{% extends "base2.html" %} + +{% block js %}{% spaceless %} + {{ block.super }} + <script type="text/javascript"> + alert("this alert shouldn't be alone!"); + </script> +{% endspaceless %}{% endblock %} diff --git a/compressor/tests/test_templates/test_block_super_multiple/base2.html b/compressor/tests/test_templates/test_block_super_multiple/base2.html index b0b2fef..c781fb5 100644 --- a/compressor/tests/test_templates/test_block_super_multiple/base2.html +++ b/compressor/tests/test_templates/test_block_super_multiple/base2.html @@ -1,3 +1,10 @@ {% extends "base.html" %} +{% block js %}{% spaceless %} + {{ block.super }} + <script type="text/javascript"> + alert("this alert should be included"); + </script> +{% endspaceless %}{% endblock %} + {% block css %}{% endblock %} diff --git a/compressor/tests/test_templates/test_complex/test_compressor_offline.html b/compressor/tests/test_templates/test_complex/test_compressor_offline.html new file mode 100644 index 0000000..6eea06e --- /dev/null +++ b/compressor/tests/test_templates/test_complex/test_compressor_offline.html @@ -0,0 +1,20 @@ +{% load compress static %}{% spaceless %} + +{% if condition %} + {% compress js%} + <script type="text/javascript">alert("{{ condition|default:"yellow" }}");</script> + {% with names=my_names %}{% spaceless %} + {% for name in names %} + <script type="text/javascript" src="{% static name %}"></script> + {% endfor %} + {% endspaceless %}{% endwith %} + {% endcompress %} +{% endif %}{% if not condition %} + {% compress js %} + <script type="text/javascript">var not_ok;</script> + {% endcompress %} +{% else %} + {% compress js %} + <script type="text/javascript">var ok = "ok";</script> + {% endcompress %} +{% endif %}{% endspaceless %} diff --git a/compressor/tests/test_templates/test_duplicate/test_compressor_offline.html b/compressor/tests/test_templates/test_duplicate/test_compressor_offline.html new file mode 100644 index 0000000..6050c8b --- /dev/null +++ b/compressor/tests/test_templates/test_duplicate/test_compressor_offline.html @@ -0,0 +1,13 @@ +{% load compress %}{% spaceless %} + +{% compress js %} + <script type="text/javascript"> + alert("Basic test"); + </script> +{% endcompress %} +{% compress js %} + <script type="text/javascript"> + alert("Basic test"); + </script> +{% endcompress %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/basic/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/basic/test_compressor_offline.html new file mode 100644 index 0000000..6e89ed2 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/basic/test_compressor_offline.html @@ -0,0 +1,8 @@ +{% spaceless %} + +{% compress js %} + <script type="text/javascript"> + alert("Basic test"); + </script> +{% endcompress %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super/base.html b/compressor/tests/test_templates_jinja2/test_block_super/base.html new file mode 100644 index 0000000..e9ca3ad --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super/base.html @@ -0,0 +1,15 @@ +{% spaceless %} +{% block js %} + <script type="text/javascript"> + alert("test using block.super"); + </script> +{% endblock %} + +{% block css %} + <style type="text/css"> + body { + background: red; + } + </style> +{% endblock %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_block_super/test_compressor_offline.html new file mode 100644 index 0000000..e1fabd8 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super/test_compressor_offline.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} + +{% block js %}{% spaceless %} + {% compress js %} + {{ super() }} + <script type="text/javascript"> + alert("this alert shouldn't be alone!"); + </script> + {% endcompress %} +{% endspaceless %}{% endblock %} + +{% block css %}{% endblock %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super_extra/base.html b/compressor/tests/test_templates_jinja2/test_block_super_extra/base.html new file mode 100644 index 0000000..e9ca3ad --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super_extra/base.html @@ -0,0 +1,15 @@ +{% spaceless %} +{% block js %} + <script type="text/javascript"> + alert("test using block.super"); + </script> +{% endblock %} + +{% block css %} + <style type="text/css"> + body { + background: red; + } + </style> +{% endblock %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super_extra/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_block_super_extra/test_compressor_offline.html new file mode 100644 index 0000000..328ccb9 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super_extra/test_compressor_offline.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block js %}{% spaceless %} + {% compress js %} + <script type="text/javascript"> + alert("this alert should be alone."); + </script> + {% endcompress %} + + {% compress js %} + {{ super() }} + <script type="text/javascript"> + alert("this alert shouldn't be alone!"); + </script> + {% endcompress %} +{% endspaceless %}{% endblock %} + +{% block css %}{% endblock %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super_multiple/base.html b/compressor/tests/test_templates_jinja2/test_block_super_multiple/base.html new file mode 100644 index 0000000..c9ee6cc --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super_multiple/base.html @@ -0,0 +1,15 @@ +{% spaceless %} +{% block js %} + <script type="text/javascript"> + alert("test using multiple inheritance and block.super"); + </script> +{% endblock %} + +{% block css %} + <style type="text/css"> + body { + background: red; + } + </style> +{% endblock %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super_multiple/base2.html b/compressor/tests/test_templates_jinja2/test_block_super_multiple/base2.html new file mode 100644 index 0000000..b0b2fef --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super_multiple/base2.html @@ -0,0 +1,3 @@ +{% extends "base.html" %} + +{% block css %}{% endblock %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super_multiple/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_block_super_multiple/test_compressor_offline.html new file mode 100644 index 0000000..accd76d --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super_multiple/test_compressor_offline.html @@ -0,0 +1,10 @@ +{% extends "base2.html" %} + +{% block js %}{% spaceless %} + {% compress js %} + {{ super() }} + <script type="text/javascript"> + alert("this alert shouldn't be alone!"); + </script> + {% endcompress %} +{% endspaceless %}{% endblock %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/base.html b/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/base.html new file mode 100644 index 0000000..c9ee6cc --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/base.html @@ -0,0 +1,15 @@ +{% spaceless %} +{% block js %} + <script type="text/javascript"> + alert("test using multiple inheritance and block.super"); + </script> +{% endblock %} + +{% block css %} + <style type="text/css"> + body { + background: red; + } + </style> +{% endblock %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/base2.html b/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/base2.html new file mode 100644 index 0000000..b0b2fef --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/base2.html @@ -0,0 +1,3 @@ +{% extends "base.html" %} + +{% block css %}{% endblock %} diff --git a/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/test_compressor_offline.html new file mode 100644 index 0000000..accd76d --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_block_super_multiple_cached/test_compressor_offline.html @@ -0,0 +1,10 @@ +{% extends "base2.html" %} + +{% block js %}{% spaceless %} + {% compress js %} + {{ super() }} + <script type="text/javascript"> + alert("this alert shouldn't be alone!"); + </script> + {% endcompress %} +{% endspaceless %}{% endblock %} diff --git a/compressor/tests/test_templates_jinja2/test_coffin/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_coffin/test_compressor_offline.html new file mode 100644 index 0000000..511ddd0 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_coffin/test_compressor_offline.html @@ -0,0 +1,11 @@ +{%- load compress -%} +{% spaceless %} + {% compress js%} + <script type="text/javascript">alert("{{ condition|default("yellow") }}"); + var ok = "{% if (25*4) is divisibleby 50 %}ok{% endif %}"; + </script> + {% with "js/one.js" as name -%} + <script type="text/javascript" src="{% static name %}"></script> + {%- endwith %} + {% endcompress %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_complex/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_complex/test_compressor_offline.html new file mode 100644 index 0000000..4707182 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_complex/test_compressor_offline.html @@ -0,0 +1,24 @@ +{% spaceless %} + +{% if condition %} + {% compress js%} + <script type="text/javascript">alert("{{ condition|default("yellow") }}");</script> + {% with names=[] -%} + {%- do names.append("js/one.js") -%} + {%- do names.append("js/nonasc.js") -%} + {% for name in names -%} + <script type="text/javascript" src="{{url_for('static', filename=name)}}"></script> + {%- endfor %} + {%- endwith %} + {% endcompress %} +{% endif %} +{% if not condition -%} + {% compress js %} + <script type="text/javascript">var not_ok;</script> + {% endcompress %} +{%- else -%} + {% compress js %} + <script type="text/javascript">var ok = "{% if (25*4) is divisibleby 50 %}ok{% endif %}";</script> + {% endcompress %} +{%- endif %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_condition/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_condition/test_compressor_offline.html new file mode 100644 index 0000000..bd1adb8 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_condition/test_compressor_offline.html @@ -0,0 +1,7 @@ +{% spaceless %} + +{% if condition %} + {% compress js%} + <script type="text/javascript">alert("{{ condition|default("yellow") }}");</script> + {% endcompress %} +{% endif %}{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_error_handling/buggy_extends.html b/compressor/tests/test_templates_jinja2/test_error_handling/buggy_extends.html new file mode 100644 index 0000000..72513f7 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_error_handling/buggy_extends.html @@ -0,0 +1,9 @@ +{% extends "buggy_template.html" %} + +{% compress css %} + <style type="text/css"> + body { + background: orange; + } + </style> +{% endcompress %} diff --git a/compressor/tests/test_templates_jinja2/test_error_handling/buggy_template.html b/compressor/tests/test_templates_jinja2/test_error_handling/buggy_template.html new file mode 100644 index 0000000..a01b899 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_error_handling/buggy_template.html @@ -0,0 +1,10 @@ +{% compress css %} + <style type="text/css"> + body { + background: pink; + } + </style> +{% endcompress %} + + +{% fail %} diff --git a/compressor/tests/test_templates_jinja2/test_error_handling/missing_extends.html b/compressor/tests/test_templates_jinja2/test_error_handling/missing_extends.html new file mode 100644 index 0000000..dc76034 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_error_handling/missing_extends.html @@ -0,0 +1,9 @@ +{% extends "missing.html" %} + +{% compress css %} + <style type="text/css"> + body { + background: purple; + } + </style> +{% endcompress %} diff --git a/compressor/tests/test_templates_jinja2/test_error_handling/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_error_handling/test_compressor_offline.html new file mode 100644 index 0000000..3ecffa5 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_error_handling/test_compressor_offline.html @@ -0,0 +1,8 @@ +{% spaceless %} + +{% compress js %} + <script type="text/javascript"> + alert("Basic test, should pass in spite of errors in other templates"); + </script> +{% endcompress %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_error_handling/with_coffeescript.html b/compressor/tests/test_templates_jinja2/test_error_handling/with_coffeescript.html new file mode 100644 index 0000000..8a53e44 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_error_handling/with_coffeescript.html @@ -0,0 +1,5 @@ +{% compress js %} + <script type="text/coffeescript" charset="utf-8"> + a = 1 + </script> +{% endcompress %} diff --git a/compressor/tests/test_templates_jinja2/test_inline_non_ascii/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_inline_non_ascii/test_compressor_offline.html new file mode 100644 index 0000000..c03b191 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_inline_non_ascii/test_compressor_offline.html @@ -0,0 +1,7 @@ +{% spaceless %} + +{% compress js, inline %} + <script type="text/javascript"> + var value = '{{ test_non_ascii_value }}'; + </script> +{% endcompress %}{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_jingo/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_jingo/test_compressor_offline.html new file mode 100644 index 0000000..d79c797 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_jingo/test_compressor_offline.html @@ -0,0 +1,11 @@ +{% spaceless %} + {% compress js%} + <script type="text/javascript">alert("{{ condition|default("yellow") }}"); + var ok = "{% if (25*4) is divisibleby 50 %}ok{% endif %}"; + var text = "{{"hello\nworld"|nl2br}}"; + </script> + {% with name="js/one.js" -%} + <script type="text/javascript" src="{{ 8|ifeq(2*4, url_for('static', name)) }}"></script> + {%- endwith %} + {% endcompress %} +{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_static_templatetag/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_static_templatetag/test_compressor_offline.html new file mode 100644 index 0000000..ed7238c --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_static_templatetag/test_compressor_offline.html @@ -0,0 +1,6 @@ +{% spaceless %} + +{% compress js %} + <script>alert('amazing');</script> + <script type="text/javascript" src="{{ url_for('static', filename="js/one.js") }}"></script> +{% endcompress %}{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_templatetag/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_templatetag/test_compressor_offline.html new file mode 100644 index 0000000..31c5d17 --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_templatetag/test_compressor_offline.html @@ -0,0 +1,7 @@ +{% spaceless %} + +{% compress js %} + <script type="text/javascript"> + alert("{{ "testtemplateTAG"|lower }}"); + </script> +{% endcompress %}{% endspaceless %} diff --git a/compressor/tests/test_templates_jinja2/test_with_context/test_compressor_offline.html b/compressor/tests/test_templates_jinja2/test_with_context/test_compressor_offline.html new file mode 100644 index 0000000..2289a5f --- /dev/null +++ b/compressor/tests/test_templates_jinja2/test_with_context/test_compressor_offline.html @@ -0,0 +1,7 @@ +{% spaceless %} + +{% compress js %} + <script type="text/javascript"> + alert("{{ content|default("Ooops!") }}"); + </script> +{% endcompress %}{% endspaceless %} diff --git a/compressor/tests/test_templatetags.py b/compressor/tests/test_templatetags.py index 151b785..db0d1b7 100644 --- a/compressor/tests/test_templatetags.py +++ b/compressor/tests/test_templatetags.py @@ -1,4 +1,4 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals import os import sys @@ -7,6 +7,7 @@ from mock import Mock from django.template import Template, Context, TemplateSyntaxError from django.test import TestCase +from django.test.utils import override_settings from compressor.conf import settings from compressor.signals import post_compress @@ -34,12 +35,12 @@ class TemplatetagTestCase(TestCase): settings.COMPRESS_ENABLED = self.old_enabled def test_empty_tag(self): - template = u"""{% load compress %}{% compress js %}{% block js %} + template = """{% load compress %}{% compress js %}{% block js %} {% endblock %}{% endcompress %}""" - self.assertEqual(u'', render(template, self.context)) + self.assertEqual('', render(template, self.context)) def test_css_tag(self): - template = u"""{% load compress %}{% compress css %} + template = """{% load compress %}{% compress css %} <link rel="stylesheet" href="{{ STATIC_URL }}css/one.css" type="text/css"> <style type="text/css">p { border:5px solid green;}</style> <link rel="stylesheet" href="{{ STATIC_URL }}css/two.css" type="text/css"> @@ -47,10 +48,8 @@ class TemplatetagTestCase(TestCase): out = css_tag("/static/CACHE/css/e41ba2cc6982.css") self.assertEqual(out, render(template, self.context)) - maxDiff = None - def test_uppercase_rel(self): - template = u"""{% load compress %}{% compress css %} + template = """{% load compress %}{% compress css %} <link rel="StyleSheet" href="{{ STATIC_URL }}css/one.css" type="text/css"> <style type="text/css">p { border:5px solid green;}</style> <link rel="StyleSheet" href="{{ STATIC_URL }}css/two.css" type="text/css"> @@ -59,7 +58,7 @@ class TemplatetagTestCase(TestCase): self.assertEqual(out, render(template, self.context)) def test_nonascii_css_tag(self): - template = u"""{% load compress %}{% compress css %} + template = """{% load compress %}{% compress css %} <link rel="stylesheet" href="{{ STATIC_URL }}css/nonasc.css" type="text/css"> <style type="text/css">p { border:5px solid green;}</style> {% endcompress %} @@ -68,40 +67,41 @@ class TemplatetagTestCase(TestCase): self.assertEqual(out, render(template, self.context)) def test_js_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} <script src="{{ STATIC_URL }}js/one.js" type="text/javascript"></script> <script type="text/javascript">obj.value = "value";</script> {% endcompress %} """ - out = u'<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' + out = '<script type="text/javascript" src="/static/CACHE/js/066cd253eada.js"></script>' self.assertEqual(out, render(template, self.context)) def test_nonascii_js_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} <script src="{{ STATIC_URL }}js/nonasc.js" type="text/javascript"></script> <script type="text/javascript">var test_value = "\u2014";</script> {% endcompress %} """ - out = u'<script type="text/javascript" src="/static/CACHE/js/e214fe629b28.js"></script>' + out = '<script type="text/javascript" src="/static/CACHE/js/e214fe629b28.js"></script>' self.assertEqual(out, render(template, self.context)) def test_nonascii_latin1_js_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} <script src="{{ STATIC_URL }}js/nonasc-latin1.js" type="text/javascript" charset="latin-1"></script> <script type="text/javascript">var test_value = "\u2014";</script> {% endcompress %} """ - out = u'<script type="text/javascript" src="/static/CACHE/js/be9e078b5ca7.js"></script>' + out = '<script type="text/javascript" src="/static/CACHE/js/be9e078b5ca7.js"></script>' self.assertEqual(out, render(template, self.context)) def test_compress_tag_with_illegal_arguments(self): - template = u"""{% load compress %}{% compress pony %} + template = """{% load compress %}{% compress pony %} <script type="pony/application">unicorn</script> {% endcompress %}""" self.assertRaises(TemplateSyntaxError, render, template, {}) + @override_settings(COMPRESS_DEBUG_TOGGLE='togglecompress') def test_debug_toggle(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} <script src="{{ STATIC_URL }}js/one.js" type="text/javascript"></script> <script type="text/javascript">obj.value = "value";</script> {% endcompress %} @@ -111,12 +111,12 @@ class TemplatetagTestCase(TestCase): GET = {settings.COMPRESS_DEBUG_TOGGLE: 'true'} context = dict(self.context, request=MockDebugRequest()) - out = u"""<script src="/static/js/one.js" type="text/javascript"></script> + out = """<script src="/static/js/one.js" type="text/javascript"></script> <script type="text/javascript">obj.value = "value";</script>""" self.assertEqual(out, render(template, context)) def test_named_compress_tag(self): - template = u"""{% load compress %}{% compress js inline foo %} + template = """{% load compress %}{% compress js inline foo %} <script type="text/javascript">obj.value = "value";</script> {% endcompress %} """ @@ -151,118 +151,94 @@ class PrecompilerTemplatetagTestCase(TestCase): settings.COMPRESS_PRECOMPILERS = self.old_precompilers def test_compress_coffeescript_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} <script type="text/coffeescript"># this is a comment.</script> {% endcompress %}""" out = script(src="/static/CACHE/js/e920d58f166d.js") self.assertEqual(out, render(template, self.context)) def test_compress_coffeescript_tag_and_javascript_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} <script type="text/coffeescript"># this is a comment.</script> <script type="text/javascript"># this too is a comment.</script> {% endcompress %}""" out = script(src="/static/CACHE/js/ef6b32a54575.js") self.assertEqual(out, render(template, self.context)) + @override_settings(COMPRESS_ENABLED=False) def test_coffeescript_and_js_tag_with_compress_enabled_equals_false(self): - self.old_enabled = settings.COMPRESS_ENABLED - settings.COMPRESS_ENABLED = False - try: - template = u"""{% load compress %}{% compress js %} - <script type="text/coffeescript"># this is a comment.</script> - <script type="text/javascript"># this too is a comment.</script> - {% endcompress %}""" - out = (script('# this is a comment.\n') + '\n' + - script('# this too is a comment.')) - self.assertEqual(out, render(template, self.context)) - finally: - settings.COMPRESS_ENABLED = self.old_enabled + template = """{% load compress %}{% compress js %} + <script type="text/coffeescript"># this is a comment.</script> + <script type="text/javascript"># this too is a comment.</script> + {% endcompress %}""" + out = (script('# this is a comment.\n') + '\n' + + script('# this too is a comment.')) + self.assertEqual(out, render(template, self.context)) + @override_settings(COMPRESS_ENABLED=False) def test_compress_coffeescript_tag_compress_enabled_is_false(self): - self.old_enabled = settings.COMPRESS_ENABLED - settings.COMPRESS_ENABLED = False - try: - template = u"""{% load compress %}{% compress js %} - <script type="text/coffeescript"># this is a comment.</script> - {% endcompress %}""" - out = script("# this is a comment.\n") - self.assertEqual(out, render(template, self.context)) - finally: - settings.COMPRESS_ENABLED = self.old_enabled + template = """{% load compress %}{% compress js %} + <script type="text/coffeescript"># this is a comment.</script> + {% endcompress %}""" + out = script("# this is a comment.\n") + self.assertEqual(out, render(template, self.context)) + @override_settings(COMPRESS_ENABLED=False) def test_compress_coffeescript_file_tag_compress_enabled_is_false(self): - self.old_enabled = settings.COMPRESS_ENABLED - settings.COMPRESS_ENABLED = False - try: - template = u""" - {% load compress %}{% compress js %} - <script type="text/coffeescript" src="{{ STATIC_URL }}js/one.coffee"> - </script> - {% endcompress %}""" + template = """ + {% load compress %}{% compress js %} + <script type="text/coffeescript" src="{{ STATIC_URL }}js/one.coffee"> + </script> + {% endcompress %}""" - out = script(src="/static/CACHE/js/one.95cfb869eead.js") - self.assertEqual(out, render(template, self.context)) - finally: - settings.COMPRESS_ENABLED = self.old_enabled + out = script(src="/static/CACHE/js/one.95cfb869eead.js") + self.assertEqual(out, render(template, self.context)) + @override_settings(COMPRESS_ENABLED=False) def test_multiple_file_order_conserved(self): - self.old_enabled = settings.COMPRESS_ENABLED - settings.COMPRESS_ENABLED = False - try: - template = u""" - {% load compress %}{% compress js %} - <script type="text/coffeescript" src="{{ STATIC_URL }}js/one.coffee"> - </script> - <script src="{{ STATIC_URL }}js/one.js"></script> - <script type="text/coffeescript" src="{{ STATIC_URL }}js/one.js"> - </script> - {% endcompress %}""" + template = """ + {% load compress %}{% compress js %} + <script type="text/coffeescript" src="{{ STATIC_URL }}js/one.coffee"> + </script> + <script src="{{ STATIC_URL }}js/one.js"></script> + <script type="text/coffeescript" src="{{ STATIC_URL }}js/one.js"> + </script> + {% endcompress %}""" - out = '\n'.join([script(src="/static/CACHE/js/one.95cfb869eead.js"), - script(scripttype="", src="/static/js/one.js"), - script(src="/static/CACHE/js/one.81a2cd965815.js")]) + out = '\n'.join([script(src="/static/CACHE/js/one.95cfb869eead.js"), + script(scripttype="", src="/static/js/one.js"), + script(src="/static/CACHE/js/one.81a2cd965815.js")]) - self.assertEqual(out, render(template, self.context)) - finally: - settings.COMPRESS_ENABLED = self.old_enabled + self.assertEqual(out, render(template, self.context)) + @override_settings(COMPRESS_ENABLED=False) def test_css_multiple_files_disabled_compression(self): - self.old_enabled = settings.COMPRESS_ENABLED - settings.COMPRESS_ENABLED = False assert(settings.COMPRESS_PRECOMPILERS) - try: - template = u""" - {% load compress %}{% compress css %} - <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/one.css"></link> - <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/two.css"></link> - {% endcompress %}""" + template = """ + {% load compress %}{% compress css %} + <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/one.css"></link> + <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/two.css"></link> + {% endcompress %}""" - out = ''.join(['<link rel="stylesheet" type="text/css" href="/static/css/one.css" />', - '<link rel="stylesheet" type="text/css" href="/static/css/two.css" />']) + out = ''.join(['<link rel="stylesheet" type="text/css" href="/static/css/one.css" />', + '<link rel="stylesheet" type="text/css" href="/static/css/two.css" />']) - self.assertEqual(out, render(template, self.context)) - finally: - settings.COMPRESS_ENABLED = self.old_enabled + self.assertEqual(out, render(template, self.context)) + @override_settings(COMPRESS_ENABLED=False) def test_css_multiple_files_mixed_precompile_disabled_compression(self): - self.old_enabled = settings.COMPRESS_ENABLED - settings.COMPRESS_ENABLED = False assert(settings.COMPRESS_PRECOMPILERS) - try: - template = u""" - {% load compress %}{% compress css %} - <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/one.css"/> - <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/two.css"/> - <link rel="stylesheet" type="text/less" href="{{ STATIC_URL }}css/url/test.css"/> - {% endcompress %}""" + template = """ + {% load compress %}{% compress css %} + <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/one.css"/> + <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/two.css"/> + <link rel="stylesheet" type="text/less" href="{{ STATIC_URL }}css/url/test.css"/> + {% endcompress %}""" - out = ''.join(['<link rel="stylesheet" type="text/css" href="/static/css/one.css" />', - '<link rel="stylesheet" type="text/css" href="/static/css/two.css" />', - '<link rel="stylesheet" href="/static/CACHE/css/test.5dddc6c2fb5a.css" type="text/css" />']) - self.assertEqual(out, render(template, self.context)) - finally: - settings.COMPRESS_ENABLED = self.old_enabled + out = ''.join(['<link rel="stylesheet" type="text/css" href="/static/css/one.css" />', + '<link rel="stylesheet" type="text/css" href="/static/css/two.css" />', + '<link rel="stylesheet" href="/static/CACHE/css/test.5dddc6c2fb5a.css" type="text/css" />']) + self.assertEqual(out, render(template, self.context)) def script(content="", src="", scripttype="text/javascript"): @@ -272,9 +248,9 @@ def script(content="", src="", scripttype="text/javascript"): >>> script('#this is a comment', scripttype="text/applescript") '<script type="text/applescript">#this is a comment</script>' """ - out_script = u'<script ' + out_script = '<script ' if scripttype: - out_script += u'type="%s" ' % scripttype + out_script += 'type="%s" ' % scripttype if src: - out_script += u'src="%s" ' % src - return out_script[:-1] + u'>%s</script>' % content + out_script += 'src="%s" ' % src + return out_script[:-1] + '>%s</script>' % content diff --git a/compressor/utils/__init__.py b/compressor/utils/__init__.py index 83a1a2a..1c3479b 100644 --- a/compressor/utils/__init__.py +++ b/compressor/utils/__init__.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals import os +from django.utils import six + from compressor.exceptions import FilterError @@ -10,15 +13,14 @@ def get_class(class_string, exception=FilterError): """ if not hasattr(class_string, '__bases__'): try: - class_string = class_string.encode('ascii') + class_string = str(class_string) mod_name, class_name = get_mod_func(class_string) - if class_name != '': - cls = getattr(__import__(mod_name, {}, {}, ['']), class_name) + if class_name: + return getattr(__import__(mod_name, {}, {}, [str('')]), class_name) except (ImportError, AttributeError): - pass - else: - return cls - raise exception('Failed to import %s' % class_string) + raise exception('Failed to import %s' % class_string) + + raise exception("Invalid class path '%s'" % class_string) def get_mod_func(callback): @@ -48,7 +50,7 @@ def find_command(cmd, paths=None, pathext=None): """ if paths is None: paths = os.environ.get('PATH', '').split(os.pathsep) - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = [paths] # check if there are funny path extensions for executables, e.g. Windows if pathext is None: diff --git a/compressor/utils/staticfiles.py b/compressor/utils/staticfiles.py index 169d427..28026f2 100644 --- a/compressor/utils/staticfiles.py +++ b/compressor/utils/staticfiles.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from django.core.exceptions import ImproperlyConfigured diff --git a/compressor/utils/stringformat.py b/compressor/utils/stringformat.py index 0cfba86..9311e78 100644 --- a/compressor/utils/stringformat.py +++ b/compressor/utils/stringformat.py @@ -6,8 +6,12 @@ An implementation of the advanced string formatting (PEP 3101). Author: Florent Xicluna """ +from __future__ import unicode_literals + import re +from django.utils import six + _format_str_re = re.compile( r'((?<!{)(?:{{)+' # '{{' r'|(?:}})+(?!})' # '}} @@ -128,7 +132,7 @@ def _format_field(value, parts, conv, spec, want_bytes=False): value = value.strftime(str(spec)) else: value = _strformat(value, spec) - if want_bytes and isinstance(value, unicode): + if want_bytes and isinstance(value, six.text_type): return str(value) return value @@ -138,9 +142,9 @@ class FormattableString(object): The method format() behaves like str.format() in python 2.6+. - >>> FormattableString(u'{a:5}').format(a=42) - ... # Same as u'{a:5}'.format(a=42) - u' 42' + >>> FormattableString('{a:5}').format(a=42) + ... # Same as '{a:5}'.format(a=42) + ' 42' """ @@ -244,13 +248,13 @@ def selftest(): import datetime F = FormattableString - assert F(u"{0:{width}.{precision}s}").format('hello world', - width=8, precision=5) == u'hello ' + assert F("{0:{width}.{precision}s}").format('hello world', + width=8, precision=5) == 'hello ' d = datetime.date(2010, 9, 7) - assert F(u"The year is {0.year}").format(d) == u"The year is 2010" - assert F(u"Tested on {0:%Y-%m-%d}").format(d) == u"Tested on 2010-09-07" - print 'Test successful' + assert F("The year is {0.year}").format(d) == "The year is 2010" + assert F("Tested on {0:%Y-%m-%d}").format(d) == "Tested on 2010-09-07" + print('Test successful') if __name__ == '__main__': selftest() diff --git a/docs/behind-the-scenes.txt b/docs/behind-the-scenes.txt index c2f87d7..0cd2a3c 100644 --- a/docs/behind-the-scenes.txt +++ b/docs/behind-the-scenes.txt @@ -18,7 +18,7 @@ even know which files are concerned actually, since it doesn't look inside the nodelist of the template block enclosed by the ``compress`` template tag. The offline cache manifest is just a json file, stored on disk inside the directory that holds the compressed files. The format of the manifest is simply -a key <=> value dictionnary, with the hash of the nodelist being the key, +a key <=> value dictionary, with the hash of the nodelist being the key, and the HTML containing the element code for the combined file or piece of code being the value. Generating the offline manifest, using the ``compress`` management command, also generates the combined files referenced in the manifest. diff --git a/docs/changelog.txt b/docs/changelog.txt index 20c1373..3828197 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -1,14 +1,51 @@ Changelog ========= +v1.4 +---- + +- Added Python 3 compatibility. + +- Added compatibility with Django 1.6.x. + +- Fixed compatibility with html5lib 1.0. + +- Added offline compression for Jinja2 with Jingo and Coffin integration. + +- Improved support for template inheritance in offline compression. + +- Made offline compression avoid compressing the same block multiple times. + +- Added a ``testenv`` target in the Makefile to make it easier to set up the + test environment. + +- Allowed data-uri filter to handle external/protocol-relative references. + +- Made ``CssCompressor`` class easier to extend. + +- Added support for explictly stating the block being ended. + +- Added rcssmin and updated rjsmin. + +- Removed implicit requirement on BeautifulSoup. + +- Made GzipCompressorFileStorage set access and modified times to the same time + as the corresponding base file. + +- Defaulted to using django's simplejson, if present. + +- Fixed CompilerFilter to always output Unicode strings. + +- Fixed windows line endings in offline compression. + v1.3 (03/18/2013) ----------------- - *Backward incompatible changes* - - Dropped support for Python 2.5. Removed ``any`` and ``walk`` compatibility + - Dropped support for Python 2.5. Removed ``any`` and ``walk`` compatibility functions in ``compressor.utils``. - + - Removed compatibility with Django 1.2 for default values of some settings: - :attr:`~COMPRESS_ROOT` no longer uses ``MEDIA_ROOT`` if ``STATIC_ROOT`` is @@ -17,7 +54,7 @@ v1.3 (03/18/2013) - :attr:`~COMPRESS_URL` no longer uses ``MEDIA_URL`` if ``STATIC_URL`` is not defined. It expects ``STATIC_URL`` to be defined instead. - - :attr:`~COMPRESS_CACHE_BACKEND` no longer uses ``CACHE_BACKEND`` and simply + - :attr:`~COMPRESS_CACHE_BACKEND` no longer uses ``CACHE_BACKEND`` and simply defaults to ``default``. - Added precompiler class support. This enables you to write custom precompilers @@ -29,8 +66,8 @@ v1.3 (03/18/2013) - Removed a ``fsync()`` call in ``CompilerFilter`` to improve performance. We already called ``self.infile.flush()`` so that call was not necessary. -- Added an extension to provide django-sekizai support. - See :ref:`django-sekizai Support <django-sekizai_support>` for more +- Added an extension to provide django-sekizai support. + See :ref:`django-sekizai Support <django-sekizai_support>` for more information. - Fixed a ``DeprecationWarning`` regarding the use of ``django.utils.hashcompat`` @@ -44,7 +81,7 @@ v1.2 - Added contributing docs. Be sure to check them out and start contributing! -- Moved CI to Travis: http://travis-ci.org/jezdez/django_compressor +- Moved CI to Travis: http://travis-ci.org/django-compressor/django-compressor - Introduced a new ``compressed`` context dictionary that is passed to the templates that are responsible for rendering the compressed snippets. diff --git a/docs/conf.py b/docs/conf.py index a0b1ab7..34552c3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,7 +42,7 @@ master_doc = 'index' # General information about the project. project = u'Django Compressor' -copyright = u'2013, Django Compressor authors' +copyright = u'2014, Django Compressor authors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -96,7 +96,8 @@ pygments_style = 'murphy' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +# html_theme = 'default' +RTD_NEW_THEME = True # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/contributing.txt b/docs/contributing.txt index ceff111..225a1ae 100644 --- a/docs/contributing.txt +++ b/docs/contributing.txt @@ -21,7 +21,7 @@ In a nutshell Here's what the contribution process looks like, in a bullet-points fashion, and only for the stuff we host on github: -#. Django Compressor is hosted on `github`_, at https://github.com/jezdez/django_compressor +#. Django Compressor is hosted on `github`_, at https://github.com/django-compressor/django-compressor #. The best method to contribute back is to create a github account, then fork the project. You can use this fork as if it was your own project, and should push your changes to it. @@ -116,9 +116,7 @@ requirements **in the virtualenv**:: $ virtualenv compressor_test $ source compressor_test/bin/activate - (compressor_test) $ pip install -e . - (compressor_test) $ pip install -r requirements/tests.txt - (compressor_test) $ pip install Django + (compressor_test) $ make testenv Then run ``make test`` to run the tests. Please note that this only tests django_compressor in the Python version you've created the virtualenv with @@ -160,7 +158,7 @@ double cookie points. Seriously. You rock. This very document is based on the contributing docs of the `django CMS`_ project. Many thanks for allowing us to steal it! -.. _Fork: http://github.com/jezdez/django_compressor +.. _Fork: http://github.com/django-compressor/django-compressor .. _Travis: http://travis-ci.org/ .. _`pull request announcment`: http://about.travis-ci.org/blog/announcing-pull-request-support/ .. _`Travis documentation`: http://about.travis-ci.org/docs/ diff --git a/docs/jinja2.txt b/docs/jinja2.txt index d23a65e..134b0b8 100644 --- a/docs/jinja2.txt +++ b/docs/jinja2.txt @@ -1,5 +1,5 @@ -Jinja2 Support -============== +Jinja2 In-Request Support +========================= Django Compressor comes with support for Jinja2_ via an extension. @@ -12,7 +12,7 @@ In order to use Django Compressor's Jinja2 extension we would need to pass import jinja2 from compressor.contrib.jinja2ext import CompressorExtension - env = jinja2.environment(extensions=[CompressorExtension]) + env = jinja2.Environment(extensions=[CompressorExtension]) From now on, you can use same code you'd normally use within Django templates:: @@ -37,5 +37,139 @@ module:: And that's it - our extension is loaded and ready to be used. + +Jinja2 Offline Compression Support +================================== +You'd need to configure ``COMPRESS_JINJA2_GET_ENVIRONMENT`` so that +Compressor can retrieve the Jinja2 environment for rendering. +This can be a lamda or function that returns a Jinja2 environment. + +Usage +----- +Run the following compress command along with an ``-engine`` parameter. The +parameter can be either jinja2 or django (default). For example, +"./manage.py compress -engine jinja2". + +Using both Django and Jinja2 templates +-------------------------------------- +There may be a chance that the Jinja2 parser is used to parse Django templates +if you have a mixture of Django and Jinja2 templates in the same location(s). +This should not be a problem since the Jinja2 parser will likely raise a +template syntax error, causing Compressor to skip the errorneous +template safely. (Vice versa for Django parser). + +A typical usage could be : + +- "./manage.py compress" for processing Django templates first, skipping + Jinja2 templates. +- "./manage.py compress -engine jinja2" for processing Jinja2 templates, + skipping Django templates. + +However, it is still recommended that you do not mix Django and Jinja2 +templates in the same project. + +Limitations +----------- +- Does not support ``{% import %}`` and similar blocks within + ``{% compress %}`` blocks. +- Does not support ``{{super()}}``. +- All other filters, globals and language constructs such as + ``{% if %}``, ``{% with %}`` and ``{% for %}`` are tested and + should run fine. + +Jinja2 templates location +------------------------- +IMPORTANT: For Compressor to discover the templates for offline compression, +there must be a template loader that implements the ``get_template_sources`` +method, and is in the ``TEMPLATE_LOADERS`` setting. + +If you're using Jinja2, you're likely to have a Jinja2 template loader in the +``TEMPLATE_LOADERS`` setting, otherwise Django won't know how to load Jinja2 +templates. You could use Jingo_ or your own custom loader. Coffin_ works +differently by providing a custom rendering method instead of a custom loader. + +Unfortunately, Jingo_ does not implement such a method in its loader; +Coffin_ does not seem to have a template loader in the first place. +Read on to understand how to make Compressor work nicely with Jingo_ +and Coffin_. + +By default, if you don't override the ``TEMPLATE_LOADERS`` setting, +it will include the app directories loader that searches for templates under +the ``templates`` directory in each app. If the app directories loader is in use +and your Jinja2 templates are in the ``<app>/templates`` directories, +Compressor will be able to find the Jinja2 templates. + +However, if you have Jinja2 templates in other location(s), you could include +the filesystem loader (``django.template.loaders.filesystem.Loader``) in the +``TEMPLATE_LOADERS`` setting and specify the custom location in the +``TEMPLATE_DIRS`` setting. + +For Jingo users +--------------- +You should configure ``TEMPLATE_LOADERS`` as such:: + + TEMPLATE_LOADERS = ( + 'jingo.Loader', + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ) + + def COMPRESS_JINJA2_GET_ENVIRONMENT(): + # TODO: ensure the CompressorExtension is installed with Jingo via + # Jingo's JINJA_CONFIG setting. + # Additional globals, filters, tests, + # and extensions used within {%compress%} blocks must be configured + # with Jingo. + from jingo import env + + return env + +This will enable the Jingo_ loader to load Jinja2 templates and the other +loaders to report the templates location(s). + +For Coffin users +---------------- +You might want to configure ``TEMPLATE_LOADERS`` as such:: + + TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ) + + def COMPRESS_JINJA2_GET_ENVIRONMENT(): + # TODO: ensure the CompressorExtension is installed with Coffin + # as described in the "In-Request Support" section above. + # Additional globals, filters, tests, + # and extensions used within {%compress%} blocks must be configured + # with Coffin. + from coffin.common import env + + return env + +Again, if you have the Jinja2 templates in the app template directories, you're +done here. Otherwise, specify the location in ``TEMPLATE_DIRS``. + +Using your custom loader +------------------------ +You should configure ``TEMPLATE_LOADERS`` as such:: + + TEMPLATE_LOADERS = ( + 'your_app.Loader', + ... other loaders (optional) ... + ) + +You could implement the `get_template_sources` method in your loader or make +use of the Django's builtin loaders to report the Jinja2 template location(s). + +Python 3 Support +---------------- +Jingo with Jinja2 are tested and work on Python 2.6, 2.7, and 3.3. +Coffin with Jinja2 are tested and work on Python 2.6 and 2.7 only. +Jinja2 alone (with custom loader) are tested and work on Python 2.6, 2.7 and +3.3 only. + + .. _Jinja2: http://jinja.pocoo.org/docs/ .. _Coffin: http://pypi.python.org/pypi/Coffin +.. _Jingo: https://jingo.readthedocs.org/en/latest/ + diff --git a/docs/quickstart.txt b/docs/quickstart.txt index d922c8b..4acfab2 100644 --- a/docs/quickstart.txt +++ b/docs/quickstart.txt @@ -32,6 +32,10 @@ Installation 'compressor.finders.CompressorFinder', ) +* Define :attr:`COMPRESS_ROOT <django.conf.settings.COMPRESS_ROOT>` in settings + if you don't have already ``STATIC_ROOT`` or if you want it in a different + folder. + .. _staticfiles: http://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/ .. _django-staticfiles: http://pypi.python.org/pypi/django-staticfiles @@ -55,17 +59,6 @@ dependencies. pip install django-appconf -- versiontools_ - - Used internally to handle versions better, this is - automatically installed when following the above - installation instructions. - - In case you're installing Django Compressor differently - (e.g. from the Git repo), make sure to install it, e.g.:: - - pip install versiontools - Optional ^^^^^^^^ @@ -104,4 +97,4 @@ Optional .. _html5lib: http://code.google.com/p/html5lib/ .. _`Slim It`: http://slimit.org/ .. _django-appconf: http://pypi.python.org/pypi/django-appconf/ -.. _versiontools: http://pypi.python.org/pypi/versiontools/
\ No newline at end of file +.. _versiontools: http://pypi.python.org/pypi/versiontools/ diff --git a/docs/settings.txt b/docs/settings.txt index c2d684c..0d3fd72 100644 --- a/docs/settings.txt +++ b/docs/settings.txt @@ -119,6 +119,18 @@ Backend settings The arguments passed to the compressor. + - ``compressor.filters.yuglify.YUglifyCSSFilter`` + + A filter that passes the CSS content to the `yUglify compressor`_. + + .. attribute:: COMPRESS_YUGLIFY_BINARY + + The yUglify compressor filesystem path. + + .. attribute:: COMPRESS_YUGLIFY_CSS_ARGUMENTS + + The arguments passed to the compressor. Defaults to --terminal. + - ``compressor.filters.cssmin.CSSMinFilter`` A filter that uses Zachary Voase's Python port of the YUI CSS compression @@ -184,6 +196,18 @@ Backend settings The arguments passed to the compressor. + - ``compressor.filters.yuglify.YUglifyJSFilter`` + + A filter that passes the JavaScript code to the `yUglify compressor`_. + + .. attribute:: COMPRESS_YUGLIFY_BINARY + + The yUglify compressor filesystem path. + + .. attribute:: COMPRESS_YUGLIFY_JS_ARGUMENTS + + The arguments passed to the compressor. + - ``compressor.filters.template.TemplateFilter`` A filter that renders the JavaScript code with Django templating system. @@ -195,6 +219,7 @@ Backend settings .. _rJSmin: http://opensource.perlig.de/rjsmin/ .. _`Google Closure compiler`: http://code.google.com/closure/compiler/ .. _`YUI compressor`: http://developer.yahoo.com/yui/compressor/ + .. _`yUglify compressor`: https://github.com/yui/yuglify .. _`Slim It`: http://slimit.org/ .. attribute:: COMPRESS_PRECOMPILERS @@ -206,7 +231,7 @@ Backend settings item: #. mimetype - The mimetype of the file or inline code should that should be compiled. + The mimetype of the file or inline code that should be compiled. #. command_or_filter The command to call on each of the files. Modern Python string @@ -334,7 +359,7 @@ Caching settings .. attribute:: COMPRESS_CACHE_BACKEND - :Default: ``"default"`` or ``CACHE_BACKEND`` + :Default: ``CACHES["default"]`` or ``CACHE_BACKEND`` The backend to use for caching, in case you want to use a different cache backend for Django Compressor. @@ -367,7 +392,7 @@ Caching settings :Default: ``10`` The amount of time (in seconds) to cache the modification timestamp of a - file. Disabled by default. Should be smaller than + file. Should be smaller than :attr:`~django.conf.settings.COMPRESS_REBUILD_TIMEOUT` and :attr:`~django.conf.settings.COMPRESS_MINT_DELAY`. @@ -393,6 +418,14 @@ Caching settings and the ``django.core.context_processors.request`` context processor. .. _RequestContext: http://docs.djangoproject.com/en/dev/ref/templates/api/#django.template.RequestContext + +.. attribute:: COMPRESS_CACHE_KEY_FUNCTION + + :Default: ``'compressor.cache.simple_cachekey'`` + + The function to use when generating the cache key. The function must take + one argument which is the partial key based on the source's hex digest. + It must return the full key as a string. Offline settings ---------------- diff --git a/requirements/tests.txt b/requirements/tests.txt index 6c27e19..775874f 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,9 +1,10 @@ flake8 -django-discover-runner coverage -unittest2 -BeautifulSoup==3.2.0 html5lib mock jinja2 -lxml
\ No newline at end of file +lxml +BeautifulSoup +unittest2 +coffin +jingo diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5e40900 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[wheel] +universal = 1 @@ -1,23 +1,31 @@ +from __future__ import print_function +import ast import os -import re import sys import codecs from fnmatch import fnmatchcase from distutils.util import convert_path from setuptools import setup, find_packages +class VersionFinder(ast.NodeVisitor): + def __init__(self): + self.version = None + + def visit_Assign(self, node): + if node.targets[0].id == '__version__': + self.version = node.value.s + def read(*parts): - return codecs.open(os.path.join(os.path.dirname(__file__), *parts)).read() + filename = os.path.join(os.path.dirname(__file__), *parts) + with codecs.open(filename, encoding='utf-8') as fp: + return fp.read() -def find_version(*file_paths): - version_file = read(*file_paths) - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", - version_file, re.M) - if version_match: - return version_match.group(1) - raise RuntimeError("Unable to find version string.") +def find_version(*parts): + finder = VersionFinder() + finder.visit(ast.parse(read(*parts))) + return finder.version # Provided as an attribute, so you can append to these instead @@ -76,9 +84,8 @@ def find_package_data(where='.', package='', if (fnmatchcase(name, pattern) or fn.lower() == pattern.lower()): bad_name = True if show_ignored: - print >> sys.stderr, ( - "Directory %s ignored by pattern %s" - % (fn, pattern)) + print("Directory %s ignored by pattern %s" % + (fn, pattern), file=sys.stderr) break if bad_name: continue @@ -97,9 +104,8 @@ def find_package_data(where='.', package='', if (fnmatchcase(name, pattern) or fn.lower() == pattern.lower()): bad_name = True if show_ignored: - print >> sys.stderr, ( - "File %s ignored by pattern %s" - % (fn, pattern)) + print("File %s ignored by pattern %s" % + (fn, pattern), file=sys.stderr) break if bad_name: continue @@ -124,8 +130,12 @@ setup( 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', 'Topic :: Internet :: WWW/HTTP', ], zip_safe=False, @@ -0,0 +1,121 @@ +[deps] +two = + flake8 + coverage + html5lib + mock + jinja2 + lxml + BeautifulSoup + unittest2 + jingo + coffin +three = + flake8 + coverage + html5lib + mock + jinja2 + lxml + BeautifulSoup4 + jingo + coffin +three_two = + flake8 + coverage + html5lib + mock + jinja2==2.6 + lxml + BeautifulSoup4 + jingo + coffin + +[tox] +envlist = + py33-1.6.X, + py32-1.6.X, + py27-1.6.X, + py26-1.6.X, + py33-1.5.X, + py32-1.5.X, + py27-1.5.X, + py26-1.5.X, + py27-1.4.X, + py26-1.4.X + +[testenv] +setenv = + CPPFLAGS=-O0 +usedevelop = true +whitelist_externals = /usr/bin/make +downloadcache = {toxworkdir}/_download/ +commands = + django-admin.py --version + make test + +[testenv:py33-1.6.X] +basepython = python3.3 +deps = + Django>=1.6,<1.7 + {[deps]three} + +[testenv:py32-1.6.X] +basepython = python3.2 +deps = + Django>=1.6,<1.7 + {[deps]three_two} + +[testenv:py27-1.6.X] +basepython = python2.7 +deps = + Django>=1.6,<1.7 + {[deps]two} + +[testenv:py26-1.6.X] +basepython = python2.6 +deps = + Django>=1.6,<1.7 + {[deps]two} + +[testenv:py33-1.5.X] +basepython = python3.3 +deps = + Django>=1.5,<1.6 + django-discover-runner + {[deps]three} + +[testenv:py32-1.5.X] +basepython = python3.2 +deps = + Django>=1.5,<1.6 + django-discover-runner + {[deps]three_two} + +[testenv:py27-1.5.X] +basepython = python2.7 +deps = + Django>=1.5,<1.6 + django-discover-runner + {[deps]two} + +[testenv:py26-1.5.X] +basepython = python2.6 +deps = + Django>=1.5,<1.6 + django-discover-runner + {[deps]two} + +[testenv:py27-1.4.X] +basepython = python2.7 +deps = + Django>=1.4,<1.5 + django-discover-runner + {[deps]two} + +[testenv:py26-1.4.X] +basepython = python2.6 +deps = + Django>=1.4,<1.5 + django-discover-runner + {[deps]two} |