Question: class Avg(FixDurationInputMixin, NumericOutputFieldMixin, Aggregate):     function = 'AVG'     name = 'Avg'<ins>    allow_distinct = True</ins>   class Count(Aggregate):

Add DISTINCT support for Avg and Sum aggregates. Description As an extension of , aggregates should be supported for other general aggregates such as Avg and Sum. Before 2.2, these aggregations just ignored the parameter, but now throw an exception. This change would just involve setting these classes as allowing DISTINCT, and could also be applied to Min and Max (although pointless).


Question: class Sum(FixDurationInputMixin, Aggregate):     function = 'SUM'     name = 'Sum'<ins>    allow_distinct = True</ins>   class Variance(NumericOutputFieldMixin, Aggregate):


Question: @deconstructible class F(Combinable):     """An object capable of resolving references to existing query objects."""    # Can the expression be used in a WHERE clause?    filterable = True      def __init__(self, name):         """diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py-- a/django/db/models/sql/query.py++ b/django/db/models/sql/query.py

Filter by window expression should raise a descriptive error. Description Django has a check that filter does not contain window expressions. But it is shallow, neither right side of the expression nor combined expressions are checked. class Employee(models.Model): grade = models.IntegerField() Employee.objects.annotate( prevgrade=Window(expression=Lag('grade')) ).filter(prevgrade=F('grade')) Employee.objects.annotate( prevgrade=Window(expression=Lag('grade')) ).filter(grade=F('prevgrade')) Employee.objects.annotate( prevgrade=Window(expression=Lag('grade')), decgrade=F('prevgrade') - Value(1) ).filter(decgrade=F('grade'))


Question: for v in value:                     self.check_query_object_type(v, opts, field) <ins>    def check_filterable(self, expression):        """Raise an error if expression cannot be used in a WHERE clause."""        if not getattr(expression, 'filterable', 'True'):            raise NotSupportedError(                expression.__class__.__name__ + ' is disallowed in the filter '                'clause.'            )        if hasattr(expression, 'get_source_expressions'):            for expr in expression.get_source_expressions():                self.check_filterable(expr)</ins>     def build_lookup(self, lookups, lhs, rhs):         """         Try to extract transforms and lookup from given lhs.


Question: raise FieldError("Cannot parse keyword query %r" % arg)         lookups, parts, reffed_expression = self.solve_lookup_type(arg)         if not getattr(reffed_expression, 'filterable', True):            raise NotSupportedError(                reffed_expression.__class__.__name__ + ' is disallowed in '                'the filter clause.'            )        self.check_filterable(reffed_expression)          if not allow_joins and len(parts) > 1:             raise FieldError("Joined field references are not permitted in this query")


Question: value = self.resolve_lookup_value(value, can_reuse, allow_joins, simple_col)         used_joins = {k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0)} <ins>        self.check_filterable(value)</ins>         clause = self.where_class()         if reffed_expression:             condition = self.build_lookup(lookups, reffed_expression, value)


Question: from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.backends.ddl_references import IndexColumns<ins>from django.db.backends.utils import strip_quotes</ins>   class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):

SQLite3 migrations can fail when used quoted dbtable. Description (last modified by Maciej Olko) If model's Meta dbtable is quoted, e.g. '"tablewithquotedname"', SQLite3 migration with this table creation with can fail with near "tablewithquotedname": syntax error. I suppose following generated query causes the error: CREATE TABLE "new"tablewithquotedname"" ("objid" integer NOT NULL PRIMARY KEY, "objnum" varchar(20) NULL, "countryid" integer NOT NULL REFERENCES "countries" ("countryid") DEFERRABLE INITIALLY DEFERRED) To reproduce table with quoted name should have at least one foreign key. Django documentation says it supports quoted names (). Quoted names can also be used with Django’s other supported database backends; except for Oracle, however, the quotes have no effect.


Question: from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.backends.ddl_references import IndexColumns<ins>from django.db.backends.utils import strip_quotes</ins>   class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):

Traceback: Traceback (most recent call last): File "…", line 82, in execute return (sql) File "…", line 382, in execute return Database.Cursor.execute(self, query) sqlite3.OperationalError: near "tablewithquotedname": syntax error The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleexecutefromcommandline() … File "…", line 234, in handle fakeinitial=fakeinitial, File "…", line 117, in migrate state = self.migrateallforwards(state, plan, fullplan, fake=fake, fakeinitial=fakeinitial) File "…", line 147, in migrateallforwards state = self.applymigration(state, migration, fake=fake, fakeinitial=fakeinitial) File "…", line 245, in applymigration state = (state, schemaeditor) File "…", line 124, in apply operation.databaseforwards(self.applabel, schemaeditor, oldstate, projectstate) File "…", line 112, in databaseforwards field, File "…", line 327, in addfield self.remaketable(model, createfield=field) File "…", line 279, in remaketable self.createmodel(newmodel) File "…", line 307, in createmodel self.execute(sql, params or None) File "…", line 137, in execute cursor.execute(sql, params) File "…", line 99, in execute return super().execute(sql, params) File "…", line 67, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "…", line 76, in executewithwrappers return executor(sql, params, many, context) File "…", line 84, in execute return (sql, params) File "…", line 89, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "…", line 82, in execute return (sql) File "…", line 382, in execute return Database.Cursor.execute(self, query) near "tablewithquoted_name": syntax error


Question: if self._field_data_type(old_field) != self._field_data_type(new_field):             self.sql_alter_column_type += ' USING %(column)s::%(type)s'         # Make ALTER TYPE with SERIAL make sense.        table = model._meta.db_table        table = strip_quotes(model._meta.db_table)         if new_type.lower() in ("serial", "bigserial"):            column = new_field.column            column = strip_quotes(new_field.column)             sequence_name = "%s_%s_seq" % (table, column)             col_type = "integer" if new_type.lower() == "serial" else "bigint"             return (diff --git a/django/db/backends/sqlite3/schema.py b/django/db/backends/sqlite3/schema.py-- a/django/db/backends/sqlite3/schema.py++ b/django/db/backends/sqlite3/schema.py


Question: if self._field_data_type(old_field) != self._field_data_type(new_field):             self.sql_alter_column_type += ' USING %(column)s::%(type)s'         # Make ALTER TYPE with SERIAL make sense.        table = model._meta.db_table        table = strip_quotes(model._meta.db_table)         if new_type.lower() in ("serial", "bigserial"):            column = new_field.column            column = strip_quotes(new_field.column)             sequence_name = "%s_%s_seq" % (table, column)             col_type = "integer" if new_type.lower() == "serial" else "bigint"             return (diff --git a/django/db/backends/sqlite3/schema.py b/django/db/backends/sqlite3/schema.py-- a/django/db/backends/sqlite3/schema.py++ b/django/db/backends/sqlite3/schema.py


Question: from django.apps.registry import Apps from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.backends.ddl_references import Statement<ins>from django.db.backends.utils import strip_quotes</ins> from django.db.models import UniqueConstraint from django.db.transaction import atomic from django.db.utils import NotSupportedError


Question: from django.apps.registry import Apps from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.backends.ddl_references import Statement<ins>from django.db.backends.utils import strip_quotes</ins> from django.db.models import UniqueConstraint from django.db.transaction import atomic from django.db.utils import NotSupportedError


Question: body_copy = copy.deepcopy(body)         meta_contents = {             'app_label': model._meta.app_label,            'db_table': 'new__%s' % model._meta.db_table,            'db_table': 'new__%s' % strip_quotes(model._meta.db_table),             'unique_together': unique_together,             'index_together': index_together,             'indexes': indexes,


Question: body_copy = copy.deepcopy(body)         meta_contents = {             'app_label': model._meta.app_label,            'db_table': 'new__%s' % model._meta.db_table,            'db_table': 'new__%s' % strip_quotes(model._meta.db_table),             'unique_together': unique_together,             'index_together': index_together,             'indexes': indexes,


Question: try:             with self.open(self.manifest_name) as manifest:                 return manifest.read().decode()        except OSError:        except FileNotFoundError:             return None      def load_manifest(self):

Cloaking PermissionErrors raised in ManifestFilesMixin.readmanifest(). Description While using the ManifestStaticFilesStorage, I encountered the ValueError shown below. <trimFile "/<some venv/site-", line 134, in url hashedname = hashednamefunc(*args) File "/<some venv/site-", line 422, in storedname raise ValueError("Missing staticfiles manifest entry for '%s'" % cleanname) ValueError: Missing staticfiles manifest entry for '' Further debugging showed that staticfilestorage.hashedfiles was empty. This was odd to me because existed on my system in its proper location in STATICROOT. Additionally, the '' existed as a key in the so I was very confused. I did more debugging of the manifest loading process and hit the root problem. Reading the manifest file raised a PermissionError because the user account running the Django app did not have proper permission to read from where the file was stored on the filesystem. Unfortunately,  catches any OSError and proceeds by returning no content. This was in a Vagrant virtual machine so I switched to the root user to bypass any permission errors and the manifest loading worked. I confirmed that I was able to load a page without any issue. I'd like to suggest that ManifestFilesMixin.read_manifest either a) not catch OSError at all to let the system fail so a developer can fix it, b) have more granular error handling of different OSError exception subclasses, or c) do some kind of logging to hint to the app developer that there was a problem. I'm making this suggestion because catching this error and proceeding left the app in an unrecoverable state. The ValueError listed at the beginning of the issue was actually a side effect of the deeper underlying PermissionError problem from attempting to read the manifest file. If there is any more info you need from me, please let me know. Thanks!


Question: from pathlib import Path  from django.conf import settingsfrom django.http import HttpResponse, HttpResponseNotFoundfrom django.http import Http404, HttpResponse, HttpResponseNotFound from django.template import Context, Engine, TemplateDoesNotExist from django.template.defaultfilters import pprintfrom django.urls import Resolver404, resolvefrom django.urls import resolve from django.utils import timezone from django.utils.datastructures import MultiValueDict from django.utils.encoding import force_str

When DEBUG is True, raising Http404 in a path converter's topython method does not result in a technical response Description This is the response I get (plain text): A server error occurred. Please contact the administrator. I understand a ValueError should be raised which tells the URL resolver "this path does not match, try next one" but Http404 is what came to my mind intuitively and the error message was not very helpful. One could also make a point that raising a Http404 should be valid way to tell the resolver "this is indeed the right path but the current parameter value does not match anything so stop what you are doing and let the handler return the 404 page (including a helpful error message when DEBUG is True instead of the default 'Django tried these URL patterns')". This would prove useful for example to implement a path converter that uses getobjector404.


Question: caller = ''     try:         resolver_match = resolve(request.path)    except Resolver404:    except Http404:         pass     else:         obj = resolver_match.func


Question: value = super().get_prep_value(value)         if value is None or isinstance(value, OuterRef):             return value        return int(value)        try:            return int(value)        except (TypeError, ValueError) as e:            raise e.__class__(                "Field '%s' expected a number but got %r." % (self.name, value),            ) from e      def contribute_to_class(self, cls, name, **kwargs):         assert not cls._meta.auto_field, "Model %s can't have more than one AutoField." % cls._meta.label

Add a helpful exception for invalid values passed to AutoField/FloatField/IntegerField. Description (last modified by Nick Pope) When a large model is updated and saved with invalid values, Django produces a traceback deep within the ORM, with no clue which field assignment caused the error. Developers are with: "TypeError: int() argument must be a string, a bytes-like object or a number, not 'tuple'" This change displays the field name which makes spotting errors a lot easier.


Question: value = super().get_prep_value(value)         if value is None:             return None        return float(value)        try:            return float(value)        except (TypeError, ValueError) as e:            raise e.__class__(                "Field '%s' expected a number but got %r." % (self.name, value),            ) from e      def get_internal_type(self):         return "FloatField"


Question: value = super().get_prep_value(value)         if value is None:             return None        return int(value)        try:            return int(value)        except (TypeError, ValueError) as e:            raise e.__class__(                "Field '%s' expected a number but got %r." % (self.name, value),            ) from e      def get_internal_type(self):         return "IntegerField"


Question: from itertools import chain  from django.apps import appsfrom django.core.checks import Error, Tags, registerfrom django.conf import settingsfrom django.core.checks import Error, Tags, Warning, register   @register(Tags.models)

Django throws error when different apps with different models have the same name table name. Description Error message: tablename: (models.E028) dbtable 'table_name' is used by multiple models: base.ModelName, app2.ModelName. We have a Base app that points to a central database and that has its own tables. We then have multiple Apps that talk to their own databases. Some share the same table names. We have used this setup for a while, but after upgrading to Django 2.2 we're getting an error saying we're not allowed 2 apps, with 2 different models to have the same table names. Is this correct behavior? We've had to roll back to Django 2.0 for now.


Question: indexes[model_index.name].append(model._meta.label)         for model_constraint in model._meta.constraints:             constraints[model_constraint.name].append(model._meta.label)    if settings.DATABASE_ROUTERS:        error_class, error_id = Warning, 'models.W035'        error_hint = (            'You have configured settings.DATABASE_ROUTERS. Verify that %s '            'are correctly routed to separate databases.'        )    else:        error_class, error_id = Error, 'models.E028'        error_hint = None     for db_table, model_labels in db_table_models.items():         if len(model_labels) != 1:            model_labels_str = ', '.join(model_labels)             errors.append(                Error(                error_class(                     "db_table '%s' is used by multiple models: %s."                    % (db_table, ', '.join(db_table_models[db_table])),                    % (db_table, model_labels_str),                     obj=db_table,                    id='models.E028',                    hint=(error_hint % model_labels_str) if error_hint else None,                    id=error_id,                 )             )     for index_name, model_labels in indexes.items():


Question: for key, value in query:         if value is None:             raise TypeError(                'Cannot encode None in a query string. Did you mean to pass '                'an empty string or omit the value?'                "Cannot encode None for key '%s' in a query string. Did you "                "mean to pass an empty string or omit the value?" % key             )         elif not doseq or isinstance(value, (str, bytes)):             query_val = value

Improve Exception Message In Test Client and urlencode() when None is passed as data. Description (last modified by Keith Gray) I am upgrading to 2.2.1 from 2.0.5 and my tests are failing due to a change in the test client:  It now throws an exception if a None value is provided in data given to a POST. I would like to propose an improvement to the message to display the offending Key and Value that generate the exception. I have a proposed change in my fork on github:


Question: for item in itr:                     if item is None:                         raise TypeError(                            'Cannot encode None in a query string. Did you '                            'mean to pass an empty string or omit the value?'                            "Cannot encode None for key '%s' in a query "                            "string. Did you mean to pass an empty string or "                            "omit the value?" % key                         )                     elif not isinstance(item, bytes):                         item = str(item)


Question: def patch_vary_headers(response, newheaders):     """     Add (or update) the "Vary" header in the given HttpResponse object.    newheaders is a list of header names that should be in "Vary". Existing    headers in "Vary" aren't removed.    newheaders is a list of header names that should be in "Vary". If headers    contains an asterisk, then "Vary" header will consist of a single asterisk    '*'. Otherwise, existing headers in "Vary" aren't removed.     """     # Note that we need to keep the original order intact, because cache     # implementations may rely on the order of the Vary contents in, say,

Allowing patchvaryheaders() caching utility to handle '' value. Description Function "patchvaryheaders", simply appends new headers to list. If view code sets Vary header to asterisk, the resulting header (after applying SessionMiddleware and LocaleMiddleware) looks like this: Vary: , Accept-Language, Cookie This is unnecessary and possible violates HTTP spec: The "Vary" header field in a response describes what parts of a request message, aside from the method, Host header field, and request target, might influence the origin server's process for selecting and representing this response. The value consists of either a single asterisk ("") or a list of header field names (case-insensitive). Vary = "" / 1#field-name (from ) I am using Django to implement REST API, so I'd like it to speak robust HTTP, that works with all present and future caching libraries, — even if widely used browsers and Nginx can correctly interpret current form of the header.


Question: existing_headers = {header.lower() for header in vary_headers}     additional_headers = [newheader for newheader in newheaders                           if newheader.lower() not in existing_headers]    response['Vary'] = ', '.join(vary_headers + additional_headers)    vary_headers += additional_headers    if '*' in vary_headers:        response['Vary'] = '*'    else:        response['Vary'] = ', '.join(vary_headers)   def has_vary_header(response, header_query):


Question: import logging<ins>import warnings</ins> from functools import update_wrapper  from django.core.exceptions import ImproperlyConfigured

Stop TemplateView automatically passing kwargs into the context Description Only TemplateView pushes to the context. ListView does not, I yet have to check others. This is inconsistency and, I think, it should be fixed.


Question: from django.template.response import TemplateResponse from django.urls import reverse from django.utils.decorators import classonlymethod<ins>from django.utils.deprecation import RemovedInDjango40Warningfrom django.utils.functional import SimpleLazyObject</ins>  logger = logging.getLogger('django.request')


Question: class TemplateView(TemplateResponseMixin, ContextMixin, View):    """    Render a template. Pass keyword arguments from the URLconf to the context.    """    """Render a template."""     def get(self, request, *args, **kwargs):        context = self.get_context_data(**kwargs)        # RemovedInDjango40Warning: when the deprecation ends, replace with:        #   context = self.get_context_data()        context_kwargs = _wrap_url_kwargs_with_deprecation_warning(kwargs)        context = self.get_context_data(**context_kwargs)         return self.render_to_response(context)  # RemovedInDjango40Warningdef _wrap_url_kwargs_with_deprecation_warning(url_kwargs):    context_kwargs = {}    for key, value in url_kwargs.items():        # Bind into function closure.        @SimpleLazyObject        def access_value(key=key, value=value):            warnings.warn(                'TemplateView passing URL kwargs to the context is '                'deprecated. Reference %s in your template through '                'view.kwargs instead.' % key,                RemovedInDjango40Warning, stacklevel=2,            )            return value        context_kwargs[key] = access_value    return context_kwargs class RedirectView(View):     """Provide a redirect on any GET request."""     permanent = False


Question: return any(child.is_summary for child in self.children)      @staticmethod    def _resolve_rhs(rhs, query, *args, **kwargs):        if hasattr(rhs, 'resolve_expression'):            rhs = rhs.resolve_expression(query, *args, **kwargs)        return rhs    def _resolve_leaf(expr, query, *args, **kwargs):        if hasattr(expr, 'resolve_expression'):            expr = expr.resolve_expression(query, *args, **kwargs)        return expr      @classmethod     def _resolve_node(cls, node, query, *args, **kwargs):         if hasattr(node, 'children'):             for child in node.children:                 cls._resolve_node(child, query, *args, **kwargs)        if hasattr(node, 'lhs'):            node.lhs = cls._resolve_leaf(node.lhs, query, *args, **kwargs)         if hasattr(node, 'rhs'):            node.rhs = cls._resolve_rhs(node.rhs, query, *args, **kwargs)            node.rhs = cls._resolve_leaf(node.rhs, query, *args, **kwargs)      def resolve_expression(self, *args, **kwargs):         clone = self.clone()

Nested OuterRef not looking on the right model for the field. Description (last modified by Aaron Lisman) I ran into this and made a test case for it. It seems similar to the test case above it. I'm not sure what's different about it and causing it to fail. Let me know if the query is just built wrong. diff --git index .. --- +++ -83,6 +83,28 class SimulationRun(models.Model): return "%s (%s to %s)" % (self.midpoint, , ) +class Customer(models.Model): name = models.TextField() + +class Item(models.Model): pass + +class Invoice(models.Model): INVOICE = 'invoice' EXPENSE = 'expense' + KINDCHOICES = ( (INVOICE, 'Invoice'), (EXPENSE, 'Expense'), ) + kind = models.CharField(choices=KINDCHOICES, maxlength=255, default=None) owner = models.ForeignKey(Customer, models.CASCADE) items = models.ManyToManyField(Item, relatedname='invoices') + class UUIDPK(models.Model): id = models.UUIDField(primarykey=True, default=uuid.uuid4) diff --git index .. --- +++ -24,7 +24,7 from import Approximate from .models import ( UUID, UUIDPK, Company, Employee, Experiment, Number, Result, SimulationRun, Time, Time, Customer, Item, Invoice ) -536,6 +536,55 class BasicExpressionsTests(TestCase): # This is a contrived example. It exercises the double OuterRef form.


Question: return any(child.is_summary for child in self.children)      @staticmethod    def _resolve_rhs(rhs, query, *args, **kwargs):        if hasattr(rhs, 'resolve_expression'):            rhs = rhs.resolve_expression(query, *args, **kwargs)        return rhs    def _resolve_leaf(expr, query, *args, **kwargs):        if hasattr(expr, 'resolve_expression'):            expr = expr.resolve_expression(query, *args, **kwargs)        return expr      @classmethod     def _resolve_node(cls, node, query, *args, **kwargs):         if hasattr(node, 'children'):             for child in node.children:                 cls._resolve_node(child, query, *args, **kwargs)        if hasattr(node, 'lhs'):            node.lhs = cls._resolve_leaf(node.lhs, query, *args, **kwargs)         if hasattr(node, 'rhs'):            node.rhs = cls._resolve_rhs(node.rhs, query, *args, **kwargs)            node.rhs = cls._resolve_leaf(node.rhs, query, *args, **kwargs)      def resolve_expression(self, *args, **kwargs):         clone = self.clone()

self.assertCountEqual(outer, [first, second, third]) def testnestedsubqueryouterref3(self): customer = (name='Test Customer') othercustomer = (name='Ohter Customer') + unexpensedinvoice = (kind=Invoice.INVOICE, owner=customer) unexpenseditem1 = () (unexpenseditem1) + partiallyexpensedinvoice = (kind=Invoice.INVOICE, owner=customer) expense1 = (kind=Invoice.EXPENSE, owner=customer) expenseditem1 = () unexpenseditem2 = () (expenseditem1, unexpenseditem2) (expenseditem1) + expensedinvoice = (kind=Invoice.INVOICE, owner=customer) (kind=Invoice.EXPENSE, owner=customer) expenseditem2 = () (expenseditem2) (expenseditem2) + otherinvoice = (kind=Invoice.INVOICE, owner=othercustomer) (unexpenseditem1) otherexpense = (kind=Invoice.EXPENSE, owner=othercustomer) (unexpenseditem1) + inner = ( kind=Invoice.EXPENSE, owner=OuterRef(OuterRef('owner')), items=OuterRef('id'), ) middle = ( invoices=OuterRef('id'), ).annotate( expensed=Exists(inner), ).filter( expensed=False, ) outer = ( kind=Invoice.INVOICE, owner=customer, ).annotate( unexpensed=Exists(middle), ) + self.assertTrue(().unexpensed) self.assertTrue(().unexpensed) self.assertFalse(().unexpensed) def testnestedsubqueryouterrefwithautofield(self): first = (time='09:00') second = (time='17:00') Cannot resolve keyword 'owner' into field. Choices are: expensed, id, invoices


Question: DEFAULT_DB_ALIAS, DJANGO_VERSION_PICKLE_KEY, DatabaseError, connection,     connections, router, transaction, )<ins>from django.db.models import NOT_PROVIDED</ins> from django.db.models.constants import LOOKUP_SEP from django.db.models.constraints import CheckConstraint, UniqueConstraint from django.db.models.deletion import CASCADE, Collector

Remove UPDATE query when saving a new model instance with a primary key that has a default Description (last modified by user0007) Using a model's instance: class Account(models.Model): id = models.UUIDField( primarykey=True, default=uuid.uuid4, editable=False ) title = models.TextField() account = Account() = "abc" () "appaccount" SET "title" = \'\', WHERE "appaccount"."id" = \'-150e-419f-b493-\'::uuid', INTO "appaccount" ("title", "id") VALUES (\'abc\', \'-214a-4798-a0fa-\'::uuid) Using a model's manager method: (title="abc") INTO "appaccount" ("title", "id") VALUES (\'abc\', \'-214a-4798-a0fa-\'::uuid) Using a model's instance with forceinsert argument: account = Account() = "abc" (forceinsert=true) INTO "appaccount" ("title", "id") VALUES (\'abc\', \'-214a-4798-a0fa-\'::uuid) Related issue?


Question: if not pk_set and (force_update or update_fields):             raise ValueError("Cannot force an update in save() with no primary key.")         updated = False<ins>        # Skip an UPDATE when adding an instance and primary key has a default.        if (            not force_insert and            self._state.adding and            self._meta.pk.default and            self._meta.pk.default is not NOT_PROVIDED        ):            force_insert = True</ins>         # If possible, try an UPDATE. If that doesn't update anything, do an INSERT.         if pk_set and not force_insert:             base_qs = cls._base_manager.using(using)


Question: import functools import inspect import re<ins>import string</ins> from importlib import import_module from urllib.parse import quote

path converters don't handle spaces well. Description This came up for someone on IRC last week, but I can't see that they raised a ticket about it. Correct: from import routetoregex routetoregex("<uuid:test") ('^(?P<test[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', {'test': at 0x1055e8c88}) Also working correctly: from import routetoregex routetoregex("<uuid:2") ImproperlyConfigured: URL route '<uuid:2' uses parameter name '2' which isn't a valid Python identifier. however, constructing a valid looking converter reference apparently hits neither the happy nor the unhappy path, and also I presume passes any system checks in place that might otherwise warn about the sensitivity: from import routetoregex routetoregex("<uuid: test") # note the preceeding space ('^\<uuid\:\ test\', {}) the regex is invalid the kwargs dictionary is (sort of rightly) empty. the same is true with "test " and "te st" Unless I'm misunderstanding the code therein, "test ", " test" and "te st" should all be hitting the invalid identifier part, and personally I feel like leading/trailing spaces at least could just be sanitised (stripped) as they're almost certainly accidental or for display purposes. Tested in a shell against master @ .


Question: For example, 'foo/<int:pk>' returns '^foo\\/(?P<pk>[0-9]+)'     and {'pk': <django.urls.converters.IntConverter>}.     """<ins>    if not set(route).isdisjoint(string.whitespace):        raise ImproperlyConfigured("URL route '%s' cannot contain whitespace." % route)</ins>     original_route = route     parts = ['^']     converters = {}


Question: part_sql = 'SELECT * FROM ({})'.format(part_sql)                     # Add parentheses when combining with compound query if not                     # already added for all compound queries.                    elif not features.supports_slicing_ordering_in_compound:                    elif (                        self.query.subquery or                        not features.supports_slicing_ordering_in_compound                    ):                         part_sql = '({})'.format(part_sql)                 parts += ((part_sql, part_args),)             except EmptyResultSet:

Can't use OuterRef in union Subquery Description When you make a QuerySet using the union method or the | operator, the QuerySet passed into the union method cannot reference OuterRef even when wrapped with Subquery. For example: cls = ( checklistisnull=False, part=OuterRef('id') ).values('checklistcustomer', 'created') ots = ( ownershiptransferisnull=False, part=OuterRef('id') ).values('ownershiptransfershiptargetcontact', 'created') return self.annotate( owner=Subquery((ots).values('owner')[:1]) ) Returns this error: ValueError This queryset contains a reference to an outer query and may only be used in a subquery. I get the same error with this statement: return self.annotate( owner=Subquery((cls | ots).values('owner')[:1]) ) (As an aside, I also get an error when I try to apply an orderby clause.)


Question: combinator_sql = self.connection.ops.set_operators[combinator]         if all and combinator == 'union':             combinator_sql += ' ALL'        braces = '({})' if features.supports_slicing_ordering_in_compound else '{}'        braces = '{}'        if not self.query.subquery and features.supports_slicing_ordering_in_compound:            braces = '({})'         sql_parts, args_parts = zip(*((braces.format(sql), args) for sql, args in parts))         result = [' {} '.format(combinator_sql).join(sql_parts)]         params = []diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py-- a/django/db/models/sql/query.py++ b/django/db/models/sql/query.py


Question: clone.bump_prefix(query)         clone.subquery = True         clone.where.resolve_expression(query, *args, **kwargs)<ins>        # Resolve combined queries.        if clone.combinator:            clone.combined_queries = tuple([                combined_query.resolve_expression(query, *args, **kwargs)                for combined_query in clone.combined_queries            ])</ins>         for key, value in clone.annotations.items():             resolved = value.resolve_expression(query, *args, **kwargs)             if hasattr(resolved, 'external_aliases'):


Question: attrs.setdefault('class', '')         attrs.update({             'data-ajax--cache': 'true',<ins>            'data-ajax--delay': 250,</ins>             'data-ajax--type': 'GET',             'data-ajax--url': self.get_url(),             'data-theme': 'admin-autocomplete',

Rate-limit autocomplete widgets Ajax requests Description (last modified by Federico Jaramillo Martínez) The current implementation of the Ajax autocomplete widget using Select2 in Django triggers a request for every key-press. This creates unnecessary load on servers. This patch rate-limit the requests by adding a delay using the option provided by Select2.


Question: from datetime import datetime, timedelta from django import forms from django.conf import settingsfrom django.contrib import messages from django.contrib.admin import FieldListFilter from django.contrib.admin.exceptions import (     DisallowedModelAdminLookup, DisallowedModelAdminToField,

Admin search with a null character crashes with "A string literal cannot contain NUL (0x00) characters." on PostgreSQL Description Input following URL to browser URL field and access. http://localhost/admin/auth/user/?q=%00 Crash with following Error.


Question: from datetime import datetime, timedelta from django import forms from django.conf import settingsfrom django.contrib import messages from django.contrib.admin import FieldListFilter from django.contrib.admin.exceptions import (     DisallowedModelAdminLookup, DisallowedModelAdminToField,

Environment: Request Method: GET Request URL: http://localhost/admin/auth/user/?q=%00 Django Version: 2.1.4 Python Version: 3.6.7 Installed Applications: ['', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'djangoextensions', 'restframework', 'select2', 'corsheaders', .......] Installed Middleware: ['django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'corsheaders.middleware.CorsMiddleware', '', '', '', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware'] Traceback: File "" in inner response = getresponse(request) File "" in getresponse response = self.processexceptionbymiddleware(e, request) File "" in getresponse response = wrappedcallback(request, callbackargs, callbackkwargs) File "" in wrapper return self.adminsite.adminview(view)(args, kwargs) File "" in wrappedview response = viewfunc(request, args, kwargs) File "" in wrappedviewfunc response = viewfunc(request, args, kwargs) File "" in inner return view(request, args, kwargs) File "" in wrapper return boundmethod(args, kwargs) File "" in wrappedview response = viewfunc(request, args, *kwargs) File "" in changelistview cl = self.getchangelistinstance(request) File "" in getchangelistinstance sortableby, File "" in init self.getresults(request) File "" in getresults resultcount = File "" in get res = [] = (instance) File "" in count return () File "" in count return () File "" in getcount number = obj.getaggregation(using, ['count'])['count'] File "" in getaggregation result = compiler.executesql(SINGLE) File "" in executesql cursor.execute(sql, params) File "" in execute return super().execute(sql, params) File "" in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "" in executewithwrappers return executor(sql, params, many, context) File "" in _execute return (sql, params) Exception Type: ValueError at /admin/auth/user/ Exception Value: A string literal cannot contain NUL (0x00) characters.


Question: ALL_VAR, ORDER_VAR, ORDER_TYPE_VAR, SEARCH_VAR, IS_POPUP_VAR, TO_FIELD_VAR)  class ChangeListSearchForm(forms.Form):    def __init__(self, *args, **kwargs):        super().__init__(*args, **kwargs)        # Populate "fields" dynamically because SEARCH_VAR is a variable:        self.fields = {            SEARCH_VAR: forms.CharField(required=False, strip=False),        } class ChangeList:    search_form_class = ChangeListSearchForm     def __init__(self, request, model, list_display, list_display_links,                  list_filter, date_hierarchy, search_fields, list_select_related,                  list_per_page, list_max_show_all, list_editable, model_admin, sortable_by):


Question: ALL_VAR, ORDER_VAR, ORDER_TYPE_VAR, SEARCH_VAR, IS_POPUP_VAR, TO_FIELD_VAR)  class ChangeListSearchForm(forms.Form):    def __init__(self, *args, **kwargs):        super().__init__(*args, **kwargs)        # Populate "fields" dynamically because SEARCH_VAR is a variable:        self.fields = {            SEARCH_VAR: forms.CharField(required=False, strip=False),        } class ChangeList:    search_form_class = ChangeListSearchForm     def __init__(self, request, model, list_display, list_display_links,                  list_filter, date_hierarchy, search_fields, list_select_related,                  list_per_page, list_max_show_all, list_editable, model_admin, sortable_by):


Question: self.sortable_by = sortable_by          # Get search parameters from the query string.<ins>        _search_form = self.search_form_class(request.GET)        if not _search_form.is_valid():            for error in _search_form.errors.values():                messages.error(request, ', '.join(error))        self.query = _search_form.cleaned_data.get(SEARCH_VAR) or ''</ins>         try:             self.page_num = int(request.GET.get(PAGE_VAR, 0))         except ValueError:


Question: self.sortable_by = sortable_by          # Get search parameters from the query string.<ins>        _search_form = self.search_form_class(request.GET)        if not _search_form.is_valid():            for error in _search_form.errors.values():                messages.error(request, ', '.join(error))        self.query = _search_form.cleaned_data.get(SEARCH_VAR) or ''</ins>         try:             self.page_num = int(request.GET.get(PAGE_VAR, 0))         except ValueError:


Question: self.list_editable = ()         else:             self.list_editable = list_editable<del>        self.query = request.GET.get(SEARCH_VAR, '')</del>         self.queryset = self.get_queryset(request)         self.get_results(request)         if self.is_popup:


Question: self.list_editable = ()         else:             self.list_editable = list_editable<del>        self.query = request.GET.get(SEARCH_VAR, '')</del>         self.queryset = self.get_queryset(request)         self.get_results(request)         if self.is_popup:


Question: self.extra = extra         super().__init__(output_field) <ins>    def __getstate__(self):        state = super().__getstate__()        state.pop('_constructor_args', None)        return state</ins>     def get_source_expressions(self):         return [self.query]

Pickling a QuerySet evaluates the querysets given to Subquery in annotate. Description I wrote a test case for modeled after the test from bug which is very similar. def testpicklesubqueryquerysetnotevaluated(self): """ Verifies that querysets passed into Subquery expressions are not evaluated when pickled """ groups = Group.objects.annotate( hasevent=models.Exists((groupid=models.OuterRef('id'))) ) with self.assertNumQueries(0): (()) The Subquery class (which is the base for Exists) only stores the underlying query object and throws the QuerySet away (as of this commit, although I don't think it worked before that). So in theory it shouldn't be pickling the QuerySet. However, the QuerySet is still stored on the instance within the constructorargs attribute by the decorator on the BaseExpression base class. So when the inner query object gets pickled, so does the QuerySet, which attempts to evaluate it. In this case, it gets the error "ValueError: This queryset contains a reference to an outer query and may only be used in a subquery." since the inner queryset is being evaluated independently when called from pickle: it calls QuerySet.getstate(). I'm not sure what the best solution is here. I made a patch that adds the following override to getstate to the SubQuery class, which fixes the problem and passes all other Django tests on my machine. I'll submit a PR shortly, but welcome any better approaches since I'm not sure what else that may effect. class Subquery(Expression): ... def getstate(self): objdict = super().getstate() ('constructorargs', None) return obj_dict


Question: is_popup = context['is_popup']     save_as = context['save_as']     show_save = context.get('show_save', True)    show_save_and_add_another = context.get('show_save_and_add_another', True)     show_save_and_continue = context.get('show_save_and_continue', True)     has_add_permission = context['has_add_permission']     has_change_permission = context['has_change_permission']     has_view_permission = context['has_view_permission']     has_editable_inline_admin_formsets = context['has_editable_inline_admin_formsets']     can_save = (has_change_permission and change) or (has_add_permission and add) or has_editable_inline_admin_formsets    can_save_and_add_another = (        has_add_permission and        not is_popup and        (not save_as or add) and        can_save and        show_save_and_add_another    )     can_save_and_continue = not is_popup and can_save and has_view_permission and show_save_and_continue     can_change = has_change_permission or has_editable_inline_admin_formsets     ctx = Context(context)

Allow hiding the "Save and Add Another" button with a context variable Description To provide better adjustability, to introduce new context var - showsaveandaddanother. E.g. if I want to hide button "Save and add another", I can just modify extra_context - write False to the variable. For other buttons - "Save" and "Save and continue editing", this already works exactly in this manner.


Question: change and context.get('show_delete', True)         ),         'show_save_as_new': not is_popup and has_change_permission and change and save_as,        'show_save_and_add_another': (            has_add_permission and not is_popup and            (not save_as or add) and can_save        ),        'show_save_and_add_another': can_save_and_add_another,         'show_save_and_continue': can_save_and_continue,         'show_save': show_save and can_save,         'show_close': not(show_save and can_save)


Question: Find named groups in `pattern` and replace them with the group name. E.g.,     1. ^(?P<a>\w+)/b/(\w+)$ ==> ^<a>/b/(\w+)$     2. ^(?P<a>\w+)/b/(?P<c>\w+)/$ ==> ^<a>/b/<c>/$<ins>    3. ^(?P<a>\w+)/b/(\w+) ==> ^<a>/b/(\w+)    4. ^(?P<a>\w+)/b/(?P<c>\w+) ==> ^<a>/b/<c></ins>     """     named_group_indices = [         (m.start(0), m.end(0), m.group(1))

simplifyregexp() doesn't replace trailing groups. Description replacenamedgroups() fails to replace the final named group if the urlpattern passed in is missing a trailing '/'. For example, with input r'entries/(?P<pk[^/.]+)/relationships/(?P<relatedfield\w+)' the "related_field" does not get properly replaced. A workaround is to tack on a '/' at the end and then it works. Code that reproduces this is attached. This function is used downstream in Django REST Framework. See issue 6888


Question: # Handle nested parentheses, e.g. '^(?P<a>(x|y))/b'.         unmatched_open_brackets, prev_char = 1, None         for idx, val in enumerate(pattern[end:]):<del>            # If brackets are balanced, the end of the string for the current            # named capture group pattern has been reached.            if unmatched_open_brackets == 0:                group_pattern_and_name.append((pattern[start:end + idx], group_name))                break</del>             # Check for unescaped `(` and `)`. They mark the start and end of a             # nested group.             if val == '(' and prev_char != '\\':


Question: elif val == ')' and prev_char != '\\':                 unmatched_open_brackets -= 1             prev_char = val<ins>            # If brackets are balanced, the end of the string for the current            # named capture group pattern has been reached.            if unmatched_open_brackets == 0:                group_pattern_and_name.append((pattern[start:end + idx + 1], group_name))                break</ins>      # Replace the string for named capture groups with their group names.     for group_pattern, group_name in group_pattern_and_name:


Question: Find unnamed groups in `pattern` and replace them with '<var>'. E.g.,     1. ^(?P<a>\w+)/b/(\w+)$ ==> ^(?P<a>\w+)/b/<var>$     2. ^(?P<a>\w+)/b/((x|y)\w+)$ ==> ^(?P<a>\w+)/b/<var>$<ins>    3. ^(?P<a>\w+)/b/(\w+) ==> ^(?P<a>\w+)/b/<var>    4. ^(?P<a>\w+)/b/((x|y)\w+) ==> ^(?P<a>\w+)/b/<var></ins>     """     unnamed_group_indices = [m.start(0) for m in unnamed_group_matcher.finditer(pattern)]     # Indices of the start of unnamed capture groups.


Question: # Handle nested parentheses, e.g. '^b/((x|y)\w+)$'.         unmatched_open_brackets, prev_char = 1, None         for idx, val in enumerate(pattern[start + 1:]):<del>            if unmatched_open_brackets == 0:                group_indices.append((start, start + 1 + idx))                break</del>             # Check for unescaped `(` and `)`. They mark the start and end of             # a nested group.             if val == '(' and prev_char != '\\':


Question: unmatched_open_brackets -= 1             prev_char = val <ins>            if unmatched_open_brackets == 0:                group_indices.append((start, start + 2 + idx))                break</ins>     # Remove unnamed group matches inside other unnamed capture groups.     group_start_end_indices = []     prev_end = None


Question: value = connection.ops.validate_autopk_value(value)         return value     def get_prep_value(self, value):        from django.db.models.expressions import OuterRef        return value if isinstance(value, OuterRef) else super().get_prep_value(value)     def contribute_to_class(self, cls, name, **kwargs):         assert not cls._meta.auto_field, (             "Model %s can't have more than one auto-generated field."diff --git a/django/db/models/fields/related_lookups.py b/django/db/models/fields/related_lookups.py-- a/django/db/models/fields/related_lookups.py++ b/django/db/models/fields/related_lookups.py

OuterRef in exclude() or ~Q() uses wrong model. Description The following test ( to tests/queries/testqscombinators) fails when trying to exclude results using OuterRef() def testexistsexclude(self): # filter() qs = Number.objects.annotate( foo=Exists( (tagscategoryid=OuterRef('pk')) ) ).filter(foo=True) print(qs) # works # exclude() qs = Number.objects.annotate( foo =Exists( Item.objects.exclude(tagscategoryid=OuterRef('pk')) ) ).filter(foo=True) print(qs) # crashes # filter(~Q()) qs = Number.objects.annotate( foo =Exists( (~Q(tags_categoryid=OuterRef('pk'))) ) ).filter(foo=True) print(qs) # crashes It results in the following error ValueError: This queryset contains a reference to an outer query and may only be used in a subquery


Question: class RelatedLookupMixin:     def get_prep_lookup(self):        if not isinstance(self.lhs, MultiColSource) and self.rhs_is_direct_value():        if not isinstance(self.lhs, MultiColSource) and not hasattr(self.rhs, 'resolve_expression'):             # If we get here, we are dealing with single-column relations.             self.rhs = get_normalized_value(self.rhs, self.lhs)[0]             # We need to run the related field's get_prep_value(). Consider casediff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py-- a/django/db/models/sql/query.py++ b/django/db/models/sql/query.py


Question: handle.         """         filter_lhs, filter_rhs = filter_expr        if isinstance(filter_rhs, F):        if isinstance(filter_rhs, OuterRef):            filter_expr = (filter_lhs, OuterRef(filter_rhs))        elif isinstance(filter_rhs, F):             filter_expr = (filter_lhs, OuterRef(filter_rhs.name))         # Generate the inner query.         query = Query(self.model)


Question: old_field_name = self.renamed_fields.get((app_label, model_name, field_name), field_name)             old_field = self.old_apps.get_model(app_label, old_model_name)._meta.get_field(old_field_name)             new_field = self.new_apps.get_model(app_label, model_name)._meta.get_field(field_name)<ins>            dependencies = []</ins>             # Implement any model renames on relations; these are handled by RenameModel             # so we need to exclude them from the comparison             if hasattr(new_field, "remote_field") and getattr(new_field.remote_field, "model", None):

Change uuid field to FK does not create dependency Description (last modified by Simon Charette) Hi! I am new in django community, so please help me, because i really dont know is it really "bug". I have a django project named "testproject" and two apps: testapp1, testapp2. It will be simpler to understand, with this example: class App1(models.Model): id = models.UUIDField(primarykey=True, unique=True, default=uuid.uuid4, editable=False, verbosename=('identifier')) text = models.CharField(maxlength=100, verbosename=('text')) anotherapp = models.UUIDField(null=True, blank=True, verbosename=('another app')) class App2(models.Model): id = models.UUIDField(primarykey=True, unique=True, default=uuid.uuid4, editable=False, verbosename=('identifier')) text = models.CharField(maxlength=100, verbosename=('text')) First model named "App1" has UUID field named "anotherapp": anotherapp = models.UUIDField(null=True, blank=True, verbosename=('another app')) After some time i change field from UUID to FK, like this: anotherapp = models.ForeignKey(App2, null=True, blank=True, ondelete=models.SETNULL, verbosename=('another app')) And as result i create new migration, but Migration class was unexpected result, because it does not create any "dependencies" for App2, because of FK. I think the correct solution will be create dependency for App2. This project use django version 2.2 and postgresql. Attach archive with sources. Project contains small test, after running him, you will get exception like this: ValueError: Related model 'testapp2.App2' cannot be resolved. So is it problem in django or maybe i dont understand something ? Here is my post in django users: !searchin/django-users/Django3A20uuid20to20does20create$20dependency%7Csort:date/django-users/-h9LZxFomLU/yz-NLi1cDgAJ Regards, Viktor Lomakin


Question: self.renamed_fields.get(rename_key + (to_field,), to_field)                         for to_field in new_field.to_fields                     ])<ins>                dependencies.extend(self._get_dependencies_for_foreign_key(new_field))</ins>             if hasattr(new_field, "remote_field") and getattr(new_field.remote_field, "through", None):                 rename_key = (                     new_field.remote_field.through._meta.app_label,


Question: name=field_name,                             field=field,                             preserve_default=preserve_default,                        )                        ),                        dependencies=dependencies,                     )                 else:                     # We cannot alter between m2m and concrete fields


Question: )             ] <ins>        choice_max_length = 0</ins>         # Expect [group_name, [value, display]]         for choices_group in self.choices:             try:

Add check to ensure maxlength fits longest choice. Description There is currently no check to ensure that Field.maxlength is large enough to fit the longest value in Field.choices. This would be very helpful as often this mistake is not noticed until an attempt is made to save a record with those values that are too long.


Question: for value, human_name in group_choices                 ):                     break                if self.max_length is not None and group_choices:                    choice_max_length = max(                        choice_max_length,                        *(len(value) for value, _ in group_choices if isinstance(value, str)),                    )             except (TypeError, ValueError):                 # No groups, choices in the form [value, display]                 value, human_name = group_name, group_choices                 if not is_value(value) or not is_value(human_name):                     break                if self.max_length is not None and isinstance(value, str):                    choice_max_length = max(choice_max_length, len(value))              # Special case: choices=['ab']             if isinstance(choices_group, str):                 break         else:            if self.max_length is not None and choice_max_length > self.max_length:                return [                    checks.Error(                        "'max_length' is too small to fit the longest value "                        "in 'choices' (%d characters)." % choice_max_length,                        obj=self,                        id='fields.E009',                    ),                ]             return []          return [


Question: yield opt      parser_actions = list(get_actions(parser))    mutually_exclusive_required_options = {        opt        for group in parser._mutually_exclusive_groups        for opt in group._group_actions if group.required    }     # Any required arguments which are passed in via **options must be passed     # to parse_args().     parse_args += [         '{}={}'.format(min(opt.option_strings), arg_options[opt.dest])        for opt in parser_actions if opt.required and opt.dest in options        for opt in parser_actions if (            opt.dest in options and            (opt.required or opt in mutually_exclusive_required_options)        )     ]     defaults = parser.parse_args(args=parse_args)     defaults = dict(defaults._get_kwargs(), **arg_options)

callcommand fails when argument of required mutually exclusive group is passed in kwargs. Description This error Error: one of the arguments --shop-id --shop is required is raised when I run callcommand('mycommand', shopid=1) the argument 'shopid' is part of a required mutually exclusive group: shop = parser.addmutuallyexclusivegroup(required=True) shop.addargument('--shop-id', nargs='?', type=int, default=None, dest='shopid') shop.addargument('--shop', nargs='?', type=str, default=None, dest='shopname') However, everything is fine when I call this command in this way: callcommand('mycommand, '--shop-id=1') In django sources I found that only those keyword arguments of callcommand are passed to the parser that are defined as required: parseargs += [ '{}={}'.format(min(opt.optionstrings), argoptions[]) for opt in parser.actions if opt.required and in options ] but in this special case both of them individually are not required, they are actually part of a group that is required. And the code of callcommand does nothing with groups defined in the parser.


Question: DEFAULT_INDEX_TABLESPACE = ''  # Default X-Frame-Options header valueX_FRAME_OPTIONS = 'SAMEORIGIN'X_FRAME_OPTIONS = 'DENY'  USE_X_FORWARDED_HOST = False USE_X_FORWARDED_PORT = Falsediff --git a/django/core/checks/security/base.py b/django/core/checks/security/base.py-- a/django/core/checks/security/base.py++ b/django/core/checks/security/base.py

Make security headers default. Description (last modified by Adam Johnson) Following my security headers talk at DjangoCon Europe and its related blog post (  ), I'd like to make Django use more of these security headers by default on new projects. They're always harder to roll out on existing projects than to just bake in to the new project template. On current master, running python check --deploy on a fresh project created with startproject yields these warnings: System check identified some issues: WARNINGS: ?: (security.W004) You have not set a value for the SECUREHSTSSECONDS setting. If your entire site is served only over SSL, you may want to consider setting a value and enabling HTTP Strict Transport Security. Be sure to read the documentation first; enabling HSTS carelessly can cause serious, irreversible problems. ?: (security.W006) Your SECURECONTENTTYPENOSNIFF setting is not set to True, so your pages will not be served with an 'X-Content-Type-Options: nosniff' header. You should consider enabling this header to prevent the browser from identifying content types incorrectly. ?: (security.W007) Your SECUREBROWSERXSSFILTER setting is not set to True, so your pages will not be served with an 'X-XSS-Protection: 1; mode=block' header. You should consider enabling this header to activate the browser's XSS filtering and help prevent XSS attacks. ?: (security.W008) Your SECURESSLREDIRECT setting is not set to True. Unless your site should be available over both SSL and non-SSL connections, you may want to either set this setting True or configure a load balancer or reverse-proxy server to redirect all connections to HTTPS. ?: (security.W012) SESSIONCOOKIESECURE is not set to True. Using a secure-only session cookie makes it more difficult for network traffic sniffers to hijack user sessions. ?: (security.W016) You have '' in your MIDDLEWARE, but you have not set CSRFCOOKIESECURE to True. Using a secure-only CSRF cookie makes it more difficult for network traffic sniffers to steal the CSRF token. ?: (security.W018) You should not have DEBUG set to True in deployment. ?


Question: DEFAULT_INDEX_TABLESPACE = ''  # Default X-Frame-Options header valueX_FRAME_OPTIONS = 'SAMEORIGIN'X_FRAME_OPTIONS = 'DENY'  USE_X_FORWARDED_HOST = False USE_X_FORWARDED_PORT = Falsediff --git a/django/core/checks/security/base.py b/django/core/checks/security/base.py-- a/django/core/checks/security/base.py++ b/django/core/checks/security/base.py

: (security.W019) You have 'django.middleware.clickjacking.XFrameOptionsMiddleware' in your MIDDLEWARE, but XFRAMEOPTIONS is not set to 'DENY'. The default is 'SAMEORIGIN', but unless there is a good reason for your site to serve other parts of itself in a frame, you should change it to 'DENY'. ?: (security.W020) ALLOWEDHOSTS must not be empty in deployment. System check identified 9 issues (0 silenced). Three of these come from security headers that we could activate by default in the settings SECURECONTENTTYPENOSNIFF, SECUREBROWSERXSSFILTER, and XFRAME_OPTIONS. I'd like to propose making them default in the startproject settings and even changing their global defaults (through a deprecation period) so they are activated by default.


Question: "You have "     "'django.middleware.clickjacking.XFrameOptionsMiddleware' in your "     "MIDDLEWARE, but X_FRAME_OPTIONS is not set to 'DENY'. "    "The default is 'SAMEORIGIN', but unless there is a good reason for "    "your site to serve other parts of itself in a frame, you should "    "change it to 'DENY'.",    "Unless there is a good reason for your site to serve other parts of "    "itself in a frame, you should change it to 'DENY'.",     id='security.W019', ) diff --git a/django/middleware/clickjacking.py b/django/middleware/clickjacking.py-- a/django/middleware/clickjacking.py++ b/django/middleware/clickjacking.py


Question: "You have "     "'django.middleware.clickjacking.XFrameOptionsMiddleware' in your "     "MIDDLEWARE, but X_FRAME_OPTIONS is not set to 'DENY'. "    "The default is 'SAMEORIGIN', but unless there is a good reason for "    "your site to serve other parts of itself in a frame, you should "    "change it to 'DENY'.",    "Unless there is a good reason for your site to serve other parts of "    "itself in a frame, you should change it to 'DENY'.",     id='security.W019', ) diff --git a/django/middleware/clickjacking.py b/django/middleware/clickjacking.py-- a/django/middleware/clickjacking.py++ b/django/middleware/clickjacking.py


Question: def get_xframe_options_value(self, request, response):         """         Get the value to set for the X_FRAME_OPTIONS header. Use the value from        the X_FRAME_OPTIONS setting, or 'SAMEORIGIN' if not set.        the X_FRAME_OPTIONS setting, or 'DENY' if not set.          This method can be overridden if needed, allowing it to vary based on         the request or response.         """        return getattr(settings, 'X_FRAME_OPTIONS', 'SAMEORIGIN').upper()        return getattr(settings, 'X_FRAME_OPTIONS', 'DENY').upper()


Question: def get_xframe_options_value(self, request, response):         """         Get the value to set for the X_FRAME_OPTIONS header. Use the value from        the X_FRAME_OPTIONS setting, or 'SAMEORIGIN' if not set.        the X_FRAME_OPTIONS setting, or 'DENY' if not set.          This method can be overridden if needed, allowing it to vary based on         the request or response.         """        return getattr(settings, 'X_FRAME_OPTIONS', 'SAMEORIGIN').upper()        return getattr(settings, 'X_FRAME_OPTIONS', 'DENY').upper()


Question: # Custom logging configuration. LOGGING = {} # Default exception reporter class used in case none has been# specifically assigned to the HttpRequest instance.DEFAULT_EXCEPTION_REPORTER = 'django.views.debug.ExceptionReporter' # Default exception reporter filter class used in case none has been # specifically assigned to the HttpRequest instance. DEFAULT_EXCEPTION_REPORTER_FILTER = 'django.views.debug.SafeExceptionReporterFilter'diff --git a/django/utils/log.py b/django/utils/log.py-- a/django/utils/log.py++ b/django/utils/log.py

Allow using ExceptionReporter subclass in Description (last modified by Carlton Gibson) allows using an ExceptionReporter subclass with AdminEmailHandler. Ideally we'd make the similar available for the 500 debug error view. Currently the use of is hardcoded. Move this to a parameter Provide an example of using, e.g., functools.partial to configure a subclass when specifying handler500. Updated for comment:5 Add ExceptionReporter to the documentation, explaining the relation between the exception reporter class and the filter it uses, and showing a simple override of gettracebackdata(). Add a DEFAULTEXCEPTIONREPORTER setting to allow specifying this. (At that point we could review deprecating DEFAULTEXCEPTIONREPORTER_FILTER, as was discussed on — but maybe that's unnecessary. Thoughts?)


Question: super().__init__()         self.include_html = include_html         self.email_backend = email_backend        self.reporter_class = import_string(reporter_class or 'django.views.debug.ExceptionReporter')        self.reporter_class = import_string(reporter_class or settings.DEFAULT_EXCEPTION_REPORTER)      def emit(self, record):         try:diff --git a/django/views/debug.py b/django/views/debug.py-- a/django/views/debug.py++ b/django/views/debug.py


Question: Create a technical server error response. The last three arguments are     the values returned from sys.exc_info() and friends.     """    reporter = ExceptionReporter(request, exc_type, exc_value, tb)    reporter = get_exception_reporter_class(request)(request, exc_type, exc_value, tb)     if request.is_ajax():         text = reporter.get_traceback_text()         return HttpResponse(text, status=status_code, content_type='text/plain; charset=utf-8')


Question: return getattr(request, 'exception_reporter_filter', default_filter)  <ins>def get_exception_reporter_class(request):    default_exception_reporter_class = import_string(settings.DEFAULT_EXCEPTION_REPORTER)    return getattr(request, 'exception_reporter_class', default_exception_reporter_class)</ins> class SafeExceptionReporterFilter:     """     Use annotations made by the sensitive_post_parameters and


Question: import hashlibfrom urllib.parse import quote  TEMPLATE_FRAGMENT_KEY_TEMPLATE = 'template.cache.%s.%s'   def make_template_fragment_key(fragment_name, vary_on=None):    if vary_on is None:        vary_on = ()    key = ':'.join(quote(str(var)) for var in vary_on)    args = hashlib.md5(key.encode())    return TEMPLATE_FRAGMENT_KEY_TEMPLATE % (fragment_name, args.hexdigest())    hasher = hashlib.md5()    if vary_on is not None:        for arg in vary_on:            hasher.update(str(arg).encode())            hasher.update(b':')    return TEMPLATE_FRAGMENT_KEY_TEMPLATE % (fragment_name, hasher.hexdigest())

Template Cache "maketemplatefragmentkey" function speed up + simplify (also discussing switch to alternate hashes) Description (last modified by Daniel) The maketemplatefragmentkey function in has the following (minor) issues: Using for varyon args, is not needed any more - it was originally to make the unhashed strings safe to send to memcached and similar restricted systems. But since the value is hashed, this is now adding nothing. (See ). Use of the MD5 hashing function is disallowed on certain (odd) systems, not being FIPS compliant. See (). Creates a string of all joined varyon args to send to the hashing function, rather than using the hashlib .update() method. Here is a version solving these, switching to SHA256, and speeding up the function quite a bit:  And PR:  And here's the repo showing performance improvement:  Which seems to be faster in every case. The downside of this is that the cache key is now different from before. The tests have been updated to the new values. There are other cache key generating functions used in other places which use MD5 still - if switching to SHA256 it would make sense to me to change those at the same time, meaning only one time invalidating keys on upgrade. Thoughts?


Question: # Set the max length and label for the "username" field.         self.username_field = UserModel._meta.get_field(UserModel.USERNAME_FIELD)        self.fields['username'].max_length = self.username_field.max_length or 254        username_max_length = self.username_field.max_length or 254        self.fields['username'].max_length = username_max_length        self.fields['username'].widget.attrs['maxlength'] = username_max_length         if self.fields['username'].label is None:             self.fields['username'].label = capfirst(self.username_field.verbose_name)

AuthenticationForm's username field doesn't set maxlength HTML attribute. Description AuthenticationForm's username field doesn't render with maxlength HTML attribute anymore. Regression introduced in and . !topic/django-developers/qnfSqro0DlA


Question: from django.db.models.sql.query import Query         if isinstance(self.rhs, Query):             if self.rhs.has_limit_one():                # The subquery must select only the pk.                self.rhs.clear_select_clause()                self.rhs.add_fields(['pk'])                if not self.rhs.has_select_fields:                    self.rhs.clear_select_clause()                    self.rhs.add_fields(['pk'])             else:                 raise ValueError(                     'The QuerySet value for an exact lookup must be limited to '

Filtering on query result overrides GROUP BY of internal query Description from import models a = (email__isnull=True).values('email').annotate(m=Max('id')).values('m') print(a.query) # good print(a[:1].query) # good b = (id=a[:1]) print(b.query) # GROUP BY U0."id" should be GROUP BY U0."email"


Question: databases). The abstraction barrier only works one way: this module has to know all about the internals of models in order to get the information it needs. """<ins>import copy</ins> import difflib import functools import inspect

Chaining selectrelated mutates original QuerySet. Description (last modified by Darren Maki) When creating a new QuerySet from an existing QuerySet that has had 'selectrelated' applied, if you apply another 'selectrelated' to the new QuerySet it will mutate the original QuerySet to also have the extra 'selectrelated'. from import models class ModelA(models.Model): pass class ModelB(models.Model): pass class ModelC(models.Model): modela = models.ForeignKey('foobar.ModelA', ondelete=models.CASCADE) modelb = models.ForeignKey('foobar.ModelB', ondelete=models.CASCADE) query1 = ModelC.objects.selectrelated('modela') print('QUERY 1:', str()) query2 = query1.selectrelated('modelb') print('QUERY 2:', str()) print('QUERY 1:', str()) if str() == str(): print('\n!!! The two queries are the same !!!\n') output QUERY 1: SELECT "foobarmodelc"."id", "foobarmodelc"."modelaid", "foobarmodelc"."modelbid", "foobarmodela"."id" FROM "foobarmodelc" INNER JOIN "foobarmodela" ON ("foobarmodelc"."modelaid" = "foobarmodela"."id") QUERY 2: SELECT "foobarmodelc"."id", "foobarmodelc"."modelaid", "foobarmodelc"."modelbid", "foobarmodela"."id", "foobarmodelb"."id" FROM "foobarmodelc" INNER JOIN "foobarmodela" ON ("foobarmodelc"."modelaid" = "foobarmodela"."id") INNER JOIN "foobarmodelb" ON ("foobarmodelc"."modelbid" = "foobarmodelb"."id") QUERY 1: SELECT "foobarmodelc"."id", "foobarmodelc"."modelaid", "foobarmodelc"."modelbid", "foobarmodela"."id", "foobarmodelb"."id" FROM "foobarmodelc" INNER JOIN "foobarmodela" ON ("foobarmodelc"."modelaid" = "foobarmodela".


Question: databases). The abstraction barrier only works one way: this module has to know all about the internals of models in order to get the information it needs. """<ins>import copy</ins> import difflib import functools import inspect

"id") INNER JOIN "foobarmodelb" ON ("foobarmodelc"."modelbid" = "foobarmodelb"."id") !!! The two queries are the same !!! The expectation is that the original QuerySet is not mutated, and the two queries are different. This behavior also happens with 'prefetchrelated'. Since the QuerySet methods call '()', and state that they return 'a new QuerySet instance', this behavior does not seem correct.


Question: obj._extra_select_cache = None         else:             obj._extra_select_cache = self._extra_select_cache.copy()<ins>        if self.select_related is not False:            # Use deepcopy because select_related stores fields in nested            # dicts.            obj.select_related = copy.deepcopy(obj.select_related)</ins>         if 'subq_aliases' in self.__dict__:             obj.subq_aliases = self.subq_aliases.copy()         obj.used_aliases = self.used_aliases.copy()


Question: obj._extra_select_cache = None         else:             obj._extra_select_cache = self._extra_select_cache.copy()<ins>        if self.select_related is not False:            # Use deepcopy because select_related stores fields in nested            # dicts.            obj.select_related = copy.deepcopy(obj.select_related)</ins>         if 'subq_aliases' in self.__dict__:             obj.subq_aliases = self.subq_aliases.copy()         obj.used_aliases = self.used_aliases.copy()


Question: def serialize(self):         enum_class = self.value.__class__         module = enum_class.__module__        v_string, v_imports = serializer_factory(self.value.value).serialize()        imports = {'import %s' % module, *v_imports}        return "%s.%s(%s)" % (module, enum_class.__name__, v_string), imports        return (            '%s.%s[%r]' % (module, enum_class.__name__, self.value.name),            {'import %s' % module},        )   class FloatSerializer(BaseSimpleSerializer):

Migrations uses value of enum object instead of its name. Description (last modified by oasl) When using Enum object as a default value for a CharField, the generated migration file uses the value of the Enum object instead of the its name. This causes a problem when using Django translation on the value of the Enum object. The problem is that, when the Enum object value get translated to the users language, the old migration files raise an error stating that the Enum does not have the corresponding value. (because the Enum value is translated to another language) Example: Let say we have this code in from enum import Enum from import gettextlazy as from import models class Status(Enum): GOOD = ('Good') # 'Good' will be translated BAD = ('Bad') # 'Bad' will be translated def str(self): return class Item(models.Model): status = models.CharField(default=Status.GOOD, maxlength=128) In the generated migration file, the code will be: ... ('status', models.CharField(default=Status('Good'), maxlength=128)) ... After the translation, 'Good' will be translated to another word and it will not be part of the Status Enum class any more, so the migration file will raise the error on the previous line: ValueError: 'Good' is not a valid Status Shouldn't the code generated by the migration uses the name of the Status Enum 'GOOD', not the value of it, since it is changeable? It should be: ('status', models.CharField(default=Status['GOOD'], max_length=128)) This will be correct regardless of the translated word


Question: fld = None             for part in field.split(LOOKUP_SEP):                 try:                    fld = _cls._meta.get_field(part)                    # pk is an alias that won't be found by opts.get_field.                    if part == 'pk':                        fld = _cls._meta.pk                    else:                        fld = _cls._meta.get_field(part)                     if fld.is_relation:                         _cls = fld.get_path_info()[-1].to_opts.model                    else:                        _cls = None                 except (FieldDoesNotExist, AttributeError):                     if fld is None or fld.get_transform(part) is None:                         errors.append(

models.E015 is raised when Meta.ordering contains "pk" of a related field. Description models.E015 is raised when Meta.ordering contains pk of a related field, e.g.: test_app.SomeModel: (models.E015) 'ordering' refers to the nonexistent field, related field, or lookup 'optionpk'. Regression in .


Question: """     def __init__(self, get_response=None):         self.cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS<ins>        self.page_timeout = None</ins>         self.key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX         self.cache_alias = settings.CACHE_MIDDLEWARE_ALIAS         self.cache = caches[self.cache_alias]

cachecontrol() "maxage" overrides cachepage() "timeout" Description If you decorate a view with both cachecontrol(maxage=3600) and cachepage(timeout=), the server side cache uses the maxage value instead of the timeout value. The comments in UpdateCacheMiddleware.processresponse() indicate it's trying to set the timeout by first looking for the max-age header before reverting to the default cachetimeout : Try to get the timeout from the "max-age" section of the "Cache- Control" header before reverting to using the default cachetimeout length. However, cachepage(timeout=) is explicitly setting the cachetimeout so that is what should be used. In summary, if a user wants the client-side cache to be shorter-lived than the server-side cache, it is currently not possible using these two decorators.


Question: if 'private' in response.get('Cache-Control', ()):             return response         # Try to get the timeout from the "max-age" section of the "Cache-        # Control" header before reverting to using the default cache_timeout        # length.        timeout = get_max_age(response)        # Page timeout takes precedence over the "max-age" and the default        # cache timeout.        timeout = self.page_timeout         if timeout is None:            timeout = self.cache_timeout        elif timeout == 0:            # max-age was set to 0, don't bother caching.            return response            # The timeout from the "max-age" section of the "Cache-Control"            # header takes precedence over the default cache timeout.            timeout = get_max_age(response)            if timeout is None:                timeout = self.cache_timeout            elif timeout == 0:                # max-age was set to 0, don't cache.                return response         patch_response_headers(response, timeout)         if timeout and response.status_code == 200:             cache_key = learn_cache_key(request, response, timeout, self.key_prefix, cache=self.cache)


Question: Also used as the hook point for the cache decorator, which is generated     using the decorator-from-middleware utility.     """    def __init__(self, get_response=None, cache_timeout=None, **kwargs):    def __init__(self, get_response=None, cache_timeout=None, page_timeout=None, **kwargs):         self.get_response = get_response         # We need to differentiate between "provided, but using default value",         # and "not provided". If the value is provided using a default, then


Question: if cache_timeout is None:             cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS         self.cache_timeout = cache_timeout        self.page_timeout = page_timeout         self.cache = caches[self.cache_alias]diff --git a/django/views/decorators/cache.py b/django/views/decorators/cache.py-- a/django/views/decorators/cache.py++ b/django/views/decorators/cache.py


Question: into account on caching -- just like the middleware does.     """     return decorator_from_middleware_with_args(CacheMiddleware)(        cache_timeout=timeout, cache_alias=cache, key_prefix=key_prefix        page_timeout=timeout, cache_alias=cache, key_prefix=key_prefix,     )


Question: import hashlib import re import time<ins>from collections import defaultdict</ins>  from django.conf import settings from django.core.cache import caches

patchcachecontrol should special case "no-cache" Description (last modified by Tim Graham) From my cursory reading of , it looks like patchcachecontrol needs to special case "no-cache". no-cache If the no-cache directive does not specify a field-name, then a cache MUST NOT use the response to satisfy a subsequent request without successful revalidation with the origin server. This allows an origin server to prevent caching even by caches that have been configured to return stale responses to client requests. If the no-cache directive does specify one or more field-names, then a cache MAY use the response to satisfy a subsequent request, subject to any other restrictions on caching. However, the specified field-name(s) MUST NOT be sent in the response to a subsequent request without successful revalidation with the origin server. This allows an origin server to prevent the re-use of certain header fields in a response, while still allowing caching of the rest of the response. For example, to integrate a site that uses "Vary: Cookie" with AWS CloudFront, one must use 'Cache-Control: no-cache="Set-Cookie"' if a response does not vary by cookie. (I've confirmed this with AWS support as of 10/31/2014). patchcachecontrol does not treat "no-cache" as a list. If you call patchcachecontrol(response, nocache="Set-Cookie") and then patchcachecontrol(response, nocache="foo"), you end up with 'Cache-Control: no-cache="foo"' Also, nocache=True should take precedence over nocache="foo" regardless of the order it is applied. I found Ticket which proposes to add "no-cache" to Just wanted to link it here since they are related.


Question: else:             return (t[0].lower(), True)     def dictvalue(t):    def dictvalue(*t):         if t[1] is True:             return t[0]         else:             return '%s=%s' % (t[0], t[1])     cc = defaultdict(set)     if response.get('Cache-Control'):        cc = cc_delim_re.split(response['Cache-Control'])        cc = dict(dictitem(el) for el in cc)    else:        cc = {}        for field in cc_delim_re.split(response['Cache-Control']):            directive, value = dictitem(field)            if directive == 'no-cache':                # no-cache supports multiple field names.                cc[directive].add(value)            else:                cc[directive] = value      # If there's already a max-age header but we're being asked to set a new     # max-age, use the minimum of the two ages. In practice this happens when


Question: del cc['public']      for (k, v) in kwargs.items():        cc[k.replace('_', '-')] = v    cc = ', '.join(dictvalue(el) for el in cc.items())        directive = k.replace('_', '-')        if directive == 'no-cache':            # no-cache supports multiple field names.            cc[directive].add(v)        else:            cc[directive] = v    directives = []    for directive, values in cc.items():        if isinstance(values, set):            if True in values:                # True takes precedence.                values = {True}            directives.extend([dictvalue(directive, value) for value in values])        else:            directives.append(dictvalue(directive, values))    cc = ', '.join(directives)     response['Cache-Control'] = cc


Question: try:         year = int(m.group('year'))         if year < 100:            if year < 70:                year += 2000            current_year = datetime.datetime.utcnow().year            current_century = current_year - (current_year % 100)            if year - (current_year % 100) > 50:                # year that appears to be more than 50 years in the future are                # interpreted as representing the past.                year += current_century - 100             else:                year += 1900                year += current_century         month = MONTHS.index(m.group('mon').lower()) + 1         day = int(m.group('day'))         hour = int(m.group('hour'))

two digit year check is incorrect Description (last modified by Ad Timmering) RFC 850 does not mention this, but in RFC 7231 (and there's something similar in RFC 2822), there's the following quote: Recipients of a timestamp value in rfc850-date format, which uses a two-digit year, MUST interpret a timestamp that appears to be more than 50 years in the future as representing the most recent year in the past that had the same last two digits. Current logic is hard coded to consider 0-69 to be in 2000-2069, and 70-99 to be 1970-1999, instead of comparing versus the current year.


Question: result = copy.copy(self)         memo[id(self)] = result         result.widget = copy.deepcopy(self.widget, memo)<ins>        result.error_messages = self.error_messages.copy()</ins>         result.validators = self.validators[:]         return result

Form Field’s deepcopy does not (deep)copy the error messages. Description The deepcopy method defined for the formfields () performs a shallow copy of self and does not include additional treatment for the error_messages dictionary. As a result, all copies of the same field share the same dictionary and any modification of either the dictionary or the error message itself for one formfield is immediately reflected on all other formfiels. This is relevant for Forms and ModelForms that modify the error messages of their fields dynamically: while each instance of the specific form (e.g., ProfileForm) is expected to have a set of fields “sealed” away from other instances of the same ProfileForm (), in fact all these instances share the same error messages, resulting in incorrectly raised errors. Confirmed for versions of Django going back to 1.11.


Question: def delete(self, key, version=None):         """        Delete a key from the cache, failing silently.        Delete a key from the cache and return whether it succeeded, failing        silently.         """         raise NotImplementedError('subclasses of BaseCache must provide a delete() method') diff --git a/django/core/cache/backends/db.py b/django/core/cache/backends/db.py-- a/django/core/cache/backends/db.py++ b/django/core/cache/backends/db.py

Make () return whether or not it suceeded. Description It can be quite useful when dealing with complex caching/locking systems or simply for logging purposes. Memcache clients already returns this value and it should be straigtforward to implement for file, inmemory, and database backend based on the number of returned rows. Redis del operation also returns the number of keys it successfully deleted so it should be implementable if it's eventually as discussed on the mailing list.


Question: return True      def delete(self, key, version=None):        self.delete_many([key], version)        self.validate_key(key)        return self._base_delete_many([self.make_key(key, version)])      def delete_many(self, keys, version=None):         key_list = []


Question: def _base_delete_many(self, keys):         if not keys:            return            return False          db = router.db_for_write(self.cache_model_class)         connection = connections[db]


Question: ),                 keys,             )        return bool(cursor.rowcount)      def has_key(self, key, version=None):         key = self.make_key(key, version=version)diff --git a/django/core/cache/backends/dummy.py b/django/core/cache/backends/dummy.py-- a/django/core/cache/backends/dummy.py++ b/django/core/cache/backends/dummy.py


Question: def delete(self, key, version=None):         key = self.make_key(key, version=version)         self.validate_key(key)        return False      def has_key(self, key, version=None):         key = self.make_key(key, version=version)diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py-- a/django/core/cache/backends/filebased.py++ b/django/core/cache/backends/filebased.py


Question: return False      def delete(self, key, version=None):        self._delete(self._key_to_file(key, version))        return self._delete(self._key_to_file(key, version))      def _delete(self, fname):         if not fname.startswith(self._dir) or not os.path.exists(fname):            return            return False         try:             os.remove(fname)         except FileNotFoundError:             # The file may have been removed by another process.            pass            return False        return True      def has_key(self, key, version=None):         fname = self._key_to_file(key, version)diff --git a/django/core/cache/backends/locmem.py b/django/core/cache/backends/locmem.py-- a/django/core/cache/backends/locmem.py++ b/django/core/cache/backends/locmem.py


Question: del self._cache[key]             del self._expire_info[key]         except KeyError:            pass            return False        return True      def delete(self, key, version=None):         key = self.make_key(key, version=version)         self.validate_key(key)         with self._lock:            self._delete(key)            return self._delete(key)      def clear(self):         with self._lock:diff --git a/django/core/cache/backends/memcached.py b/django/core/cache/backends/memcached.py-- a/django/core/cache/backends/memcached.py++ b/django/core/cache/backends/memcached.py


Question: def delete(self, key, version=None):         key = self.make_key(key, version=version)        self._cache.delete(key)        return bool(self._cache.delete(key))      def get_many(self, keys, version=None):         key_map = {self.make_key(key, version=version): key for key in keys}


Question: return default         return val <ins>    def delete(self, key, version=None):        # python-memcached's delete() returns True when key doesn't exist.        # https://github.com/linsomniac/python-memcached/issues/170        # Call _deletetouch() without the NOT_FOUND in expected results.        key = self.make_key(key, version=version)        return bool(self._cache._deletetouch([b'DELETED'], 'delete', key))</ins>  class PyLibMCCache(BaseMemcachedCache):     "An implementation of a cache binding using pylibmc"


Question: except models.ProtectedError as e:             self.protected.update(e.protected_objects)     def related_objects(self, related, objs):        qs = super().related_objects(related, objs)        return qs.select_related(related.field.name)    def related_objects(self, related_model, related_fields, objs):        qs = super().related_objects(related_model, related_fields, objs)        return qs.select_related(*[related_field.name for related_field in related_fields])      def _nested(self, obj, seen, format_callback):         if obj in seen:diff --git a/django/db/models/deletion.py b/django/db/models/deletion.py-- a/django/db/models/deletion.py++ b/django/db/models/deletion.py

Combine fast delete queries Description When emulating ON DELETE CASCADE via ondelete=models.CASCADE the deletion.Collector will try to perform fast queries which are DELETE FROM table WHERE IN .... There's a few conditions required for this fast path to be taken but when this happens the collection logic should combine such queries by table to reduce the number of roundtrips to the database. For example, given the following models class Person(models.Model): friends = models.ManyToManyField('self') class User(models.Model): pass class Entry(models.Model): createdby = models.ForeignKey(User) updatedby = models.ForeignKey(User) Issuing a () or () will result in 3 queries of the form DELETE FROM personfriends WHERE fromid = :id DELETE FROM personfriends WHERE toid = :id DELETE FROM person WHERE id = :id DELETE FROM entry WHERE createdbyid = :id DELETE FROM entry WHERE updatedby = :id DELETRE FROM user WHERE id = :id But both queries (or N queries depending on the number of foreign relationships) can be combined into a single one by using OR DELETE FROM personfriends WHERE fromid = :id OR toid = :id DELETE FROM person WHERE id = :id DELETE FROM entry WHERE createdbyid = :id OR updatedby = :id DELETE FROM user WHERE id = :id


Question: from collections import Counterimport operatorfrom collections import Counter, defaultdictfrom functools import partial, reduce from itertools import chain from operator import attrgetter  from django.db import IntegrityError, connections, transactionfrom django.db.models import signals, sqlfrom django.db.models import query_utils, signals, sql   class ProtectedError(IntegrityError):


Question: def __init__(self, using):         self.using = using         # Initially, {model: {instances}}, later values become lists.        self.data = {}        self.field_updates = {}  # {model: {(field, value): {instances}}}        self.data = defaultdict(set)        # {model: {(field, value): {instances}}}        self.field_updates = defaultdict(partial(defaultdict, set))         # fast_deletes is a list of queryset-likes that can be deleted without         # fetching the objects into memory.         self.fast_deletes = []


Question: # should be included, as the dependencies exist only between actual         # database tables; proxy models are represented here by their concrete         # parent.        self.dependencies = {}  # {model: {models}}        self.dependencies = defaultdict(set)  # {model: {models}}      def add(self, objs, source=None, nullable=False, reverse_dependency=False):         """


Question: return []         new_objs = []         model = objs[0].__class__        instances = self.data.setdefault(model, set())        instances = self.data[model]         for obj in objs:             if obj not in instances:                 new_objs.append(obj)


Question: if source is not None and not nullable:             if reverse_dependency:                 source, model = model, source            self.dependencies.setdefault(                source._meta.concrete_model, set()).add(model._meta.concrete_model)            self.dependencies[source._meta.concrete_model].add(model._meta.concrete_model)         return new_objs      def add_field_update(self, field, value, objs):


Question: if not objs:             return         model = objs[0].__class__        self.field_updates.setdefault(            model, {}).setdefault(            (field, value), set()).update(objs)        self.field_updates[model][field, value].update(objs)      def _has_signal_listeners(self, model):         return (


Question: if from_field and from_field.remote_field.on_delete is not CASCADE:             return False         if hasattr(objs, '_meta'):            model = type(objs)            model = objs._meta.model         elif hasattr(objs, 'model') and hasattr(objs, '_raw_delete'):             model = objs.model         else:


Question: )         )     def get_del_batches(self, objs, field):    def get_del_batches(self, objs, fields):         """         Return the objs in suitably sized batches for the used connection.         """        field_names = [field.name for field in fields]         conn_batch_size = max(            connections[self.using].ops.bulk_batch_size([field.name], objs), 1)            connections[self.using].ops.bulk_batch_size(field_names, objs), 1)         if len(objs) > conn_batch_size:             return [objs[i:i + conn_batch_size]                     for i in range(0, len(objs), conn_batch_size)]


Question: source_attr=ptr.remote_field.related_name,                                  collect_related=False,                                  reverse_dependency=True)        if collect_related:            if keep_parents:                parents = set(model._meta.get_parent_list())            for related in get_candidate_relations_to_delete(model._meta):                # Preserve parent reverse relationships if keep_parents=True.                if keep_parents and related.model in parents:                    continue                field = related.field                if field.remote_field.on_delete == DO_NOTHING:                    continue                batches = self.get_del_batches(new_objs, field)                for batch in batches:                    sub_objs = self.related_objects(related, batch)                    if self.can_fast_delete(sub_objs, from_field=field):                        self.fast_deletes.append(sub_objs)                    else:                        related_model = related.related_model                        # Non-referenced fields can be deferred if no signal                        # receivers are connected for the related model as                        # they'll never be exposed to the user. Skip field                        # deferring when some relationships are select_related                        # as interactions between both features are hard to                        # get right. This should only happen in the rare                        # cases where .related_objects is overridden anyway.                        if not (sub_objs.query.select_related or self._has_signal_listeners(related_model)):                            referenced_fields = set(chain.from_iterable(                                (rf.attname for rf in rel.field.foreign_related_fields)                                for rel in get_candidate_relations_to_delete(related_model._meta)                            ))                            sub_objs = sub_objs.only(*tuple(referenced_fields))                        if sub_objs:                            field.remote_field.on_delete(self, field, sub_objs, self.using)            for field in model._meta.private_fields:                if hasattr(field, 'bulk_related_objects'):                    # It's something like generic foreign key.                    sub_objs = field.bulk_related_objects(new_objs, self.using)                    self.collect(sub_objs, source=model, nullable=True)    def related_objects(self, related, objs):        if not collect_related:            return        if keep_parents:            parents = set(model._meta.get_parent_list())        model_fast_deletes = defaultdict(list)        for related in get_candidate_relations_to_delete(model._meta):            # Preserve parent reverse relationships if keep_parents=True.            if keep_parents and related.model in parents:                continue            field = related.field            if field.remote_field.on_delete == DO_NOTHING:                continue            related_model = related.related_model            if self.can_fast_delete(related_model, from_field=field):                model_fast_deletes[related_model].append(field)                continue            batches = self.get_del_batches(new_objs, [field])            for batch in batches:                sub_objs = self.related_objects(related_model, [field], batch)                # Non-referenced fields can be deferred if no signal receivers                # are connected for the related model as they'll never be                # exposed to the user. Skip field deferring when some                # relationships are select_related as interactions between both                # features are hard to get right. This should only happen in                # the rare cases where .related_objects is overridden anyway.                if not (sub_objs.query.select_related or self._has_signal_listeners(related_model)):                    referenced_fields = set(chain.from_iterable(                        (rf.attname for rf in rel.field.foreign_related_fields)                        for rel in get_candidate_relations_to_delete(related_model._meta)                    ))                    sub_objs = sub_objs.only(*tuple(referenced_fields))                if sub_objs:                    field.remote_field.on_delete(self, field, sub_objs, self.using)        for related_model, related_fields in model_fast_deletes.items():            batches = self.get_del_batches(new_objs, related_fields)            for batch in batches:                sub_objs = self.related_objects(related_model, related_fields, batch)                self.fast_deletes.append(sub_objs)        for field in model._meta.private_fields:            if hasattr(field, 'bulk_related_objects'):                # It's something like generic foreign key.                sub_objs = field.bulk_related_objects(new_objs, self.using)                self.collect(sub_objs, source=model, nullable=True)    def related_objects(self, related_model, related_fields, objs):         """        Get a QuerySet of objects related to `objs` via the relation `related`.        Get a QuerySet of the related model to objs via related fields.         """        return related.related_model._base_manager.using(self.using).filter(            **{"%s__in" % related.field.name: objs}        )        predicate = reduce(operator.or_, (            query_utils.Q(**{'%s__in' % related_field.name: objs})            for related_field in related_fields        ))        return related_model._base_manager.using(self.using).filter(predicate)      def instances_with_model(self):         for model, instances in self.data.items():