base.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363
  1. """
  2. MySQL database backend for Django.
  3. Requires mysqlclient: https://pypi.org/project/mysqlclient/
  4. """
  5. import re
  6. from django.core.exceptions import ImproperlyConfigured
  7. from django.db import utils
  8. from django.db.backends import utils as backend_utils
  9. from django.db.backends.base.base import BaseDatabaseWrapper
  10. from django.utils.asyncio import async_unsafe
  11. from django.utils.functional import cached_property
  12. try:
  13. import MySQLdb as Database
  14. except ImportError as err:
  15. raise ImproperlyConfigured(
  16. 'Error loading MySQLdb module.\n'
  17. 'Did you install mysqlclient?'
  18. ) from err
  19. from MySQLdb.constants import CLIENT, FIELD_TYPE # isort:skip
  20. from MySQLdb.converters import conversions # isort:skip
  21. # Some of these import MySQLdb, so import them after checking if it's installed.
  22. from .client import DatabaseClient # isort:skip
  23. from .creation import DatabaseCreation # isort:skip
  24. from .features import DatabaseFeatures # isort:skip
  25. from .introspection import DatabaseIntrospection # isort:skip
  26. from .operations import DatabaseOperations # isort:skip
  27. from .schema import DatabaseSchemaEditor # isort:skip
  28. from .validation import DatabaseValidation # isort:skip
  29. version = Database.version_info
  30. # if version < (1, 3, 13):
  31. # raise ImproperlyConfigured('mysqlclient 1.3.13 or newer is required; you have %s.' % Database.__version__)
  32. # MySQLdb returns TIME columns as timedelta -- they are more like timedelta in
  33. # terms of actual behavior as they are signed and include days -- and Django
  34. # expects time.
  35. django_conversions = {
  36. **conversions,
  37. **{FIELD_TYPE.TIME: backend_utils.typecast_time},
  38. }
  39. # This should match the numerical portion of the version numbers (we can treat
  40. # versions like 5.0.24 and 5.0.24a as the same).
  41. server_version_re = re.compile(r'(\d{1,2})\.(\d{1,2})\.(\d{1,2})')
  42. class CursorWrapper:
  43. """
  44. A thin wrapper around MySQLdb's normal cursor class that catches particular
  45. exception instances and reraises them with the correct types.
  46. Implemented as a wrapper, rather than a subclass, so that it isn't stuck
  47. to the particular underlying representation returned by Connection.cursor().
  48. """
  49. codes_for_integrityerror = (
  50. 1048, # Column cannot be null
  51. 1690, # BIGINT UNSIGNED value is out of range
  52. 3819, # CHECK constraint is violated
  53. 4025, # CHECK constraint failed
  54. )
  55. def __init__(self, cursor):
  56. self.cursor = cursor
  57. def execute(self, query, args=None):
  58. try:
  59. # args is None means no string interpolation
  60. return self.cursor.execute(query, args)
  61. except Database.OperationalError as e:
  62. # Map some error codes to IntegrityError, since they seem to be
  63. # misclassified and Django would prefer the more logical place.
  64. if e.args[0] in self.codes_for_integrityerror:
  65. raise utils.IntegrityError(*tuple(e.args))
  66. raise
  67. def executemany(self, query, args):
  68. try:
  69. return self.cursor.executemany(query, args)
  70. except Database.OperationalError as e:
  71. # Map some error codes to IntegrityError, since they seem to be
  72. # misclassified and Django would prefer the more logical place.
  73. if e.args[0] in self.codes_for_integrityerror:
  74. raise utils.IntegrityError(*tuple(e.args))
  75. raise
  76. def __getattr__(self, attr):
  77. return getattr(self.cursor, attr)
  78. def __iter__(self):
  79. return iter(self.cursor)
  80. class DatabaseWrapper(BaseDatabaseWrapper):
  81. vendor = 'mysql'
  82. # This dictionary maps Field objects to their associated MySQL column
  83. # types, as strings. Column-type strings can contain format strings; they'll
  84. # be interpolated against the values of Field.__dict__ before being output.
  85. # If a column type is set to None, it won't be included in the output.
  86. data_types = {
  87. 'AutoField': 'integer AUTO_INCREMENT',
  88. 'BigAutoField': 'bigint AUTO_INCREMENT',
  89. 'BinaryField': 'longblob',
  90. 'BooleanField': 'bool',
  91. 'CharField': 'varchar(%(max_length)s)',
  92. 'DateField': 'date',
  93. 'DateTimeField': 'datetime(6)',
  94. 'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)',
  95. 'DurationField': 'bigint',
  96. 'FileField': 'varchar(%(max_length)s)',
  97. 'FilePathField': 'varchar(%(max_length)s)',
  98. 'FloatField': 'double precision',
  99. 'IntegerField': 'integer',
  100. 'BigIntegerField': 'bigint',
  101. 'IPAddressField': 'char(15)',
  102. 'GenericIPAddressField': 'char(39)',
  103. 'NullBooleanField': 'bool',
  104. 'OneToOneField': 'integer',
  105. 'PositiveIntegerField': 'integer UNSIGNED',
  106. 'PositiveSmallIntegerField': 'smallint UNSIGNED',
  107. 'SlugField': 'varchar(%(max_length)s)',
  108. 'SmallAutoField': 'smallint AUTO_INCREMENT',
  109. 'SmallIntegerField': 'smallint',
  110. 'TextField': 'longtext',
  111. 'TimeField': 'time(6)',
  112. 'UUIDField': 'char(32)',
  113. }
  114. # For these data types:
  115. # - MySQL < 8.0.13 and MariaDB < 10.2.1 don't accept default values and
  116. # implicitly treat them as nullable
  117. # - all versions of MySQL and MariaDB don't support full width database
  118. # indexes
  119. _limited_data_types = (
  120. 'tinyblob', 'blob', 'mediumblob', 'longblob', 'tinytext', 'text',
  121. 'mediumtext', 'longtext', 'json',
  122. )
  123. operators = {
  124. 'exact': '= %s',
  125. 'iexact': 'LIKE %s',
  126. 'contains': 'LIKE BINARY %s',
  127. 'icontains': 'LIKE %s',
  128. 'gt': '> %s',
  129. 'gte': '>= %s',
  130. 'lt': '< %s',
  131. 'lte': '<= %s',
  132. 'startswith': 'LIKE BINARY %s',
  133. 'endswith': 'LIKE BINARY %s',
  134. 'istartswith': 'LIKE %s',
  135. 'iendswith': 'LIKE %s',
  136. }
  137. # The patterns below are used to generate SQL pattern lookup clauses when
  138. # the right-hand side of the lookup isn't a raw string (it might be an expression
  139. # or the result of a bilateral transformation).
  140. # In those cases, special characters for LIKE operators (e.g. \, *, _) should be
  141. # escaped on database side.
  142. #
  143. # Note: we use str.format() here for readability as '%' is used as a wildcard for
  144. # the LIKE operator.
  145. pattern_esc = r"REPLACE(REPLACE(REPLACE({}, '\\', '\\\\'), '%%', '\%%'), '_', '\_')"
  146. pattern_ops = {
  147. 'contains': "LIKE BINARY CONCAT('%%', {}, '%%')",
  148. 'icontains': "LIKE CONCAT('%%', {}, '%%')",
  149. 'startswith': "LIKE BINARY CONCAT({}, '%%')",
  150. 'istartswith': "LIKE CONCAT({}, '%%')",
  151. 'endswith': "LIKE BINARY CONCAT('%%', {})",
  152. 'iendswith': "LIKE CONCAT('%%', {})",
  153. }
  154. isolation_levels = {
  155. 'read uncommitted',
  156. 'read committed',
  157. 'repeatable read',
  158. 'serializable',
  159. }
  160. Database = Database
  161. SchemaEditorClass = DatabaseSchemaEditor
  162. # Classes instantiated in __init__().
  163. client_class = DatabaseClient
  164. creation_class = DatabaseCreation
  165. features_class = DatabaseFeatures
  166. introspection_class = DatabaseIntrospection
  167. ops_class = DatabaseOperations
  168. validation_class = DatabaseValidation
  169. def get_connection_params(self):
  170. kwargs = {
  171. 'conv': django_conversions,
  172. 'charset': 'utf8',
  173. }
  174. settings_dict = self.settings_dict
  175. if settings_dict['USER']:
  176. kwargs['user'] = settings_dict['USER']
  177. if settings_dict['NAME']:
  178. kwargs['db'] = settings_dict['NAME']
  179. if settings_dict['PASSWORD']:
  180. kwargs['passwd'] = settings_dict['PASSWORD']
  181. if settings_dict['HOST'].startswith('/'):
  182. kwargs['unix_socket'] = settings_dict['HOST']
  183. elif settings_dict['HOST']:
  184. kwargs['host'] = settings_dict['HOST']
  185. if settings_dict['PORT']:
  186. kwargs['port'] = int(settings_dict['PORT'])
  187. # We need the number of potentially affected rows after an
  188. # "UPDATE", not the number of changed rows.
  189. kwargs['client_flag'] = CLIENT.FOUND_ROWS
  190. # Validate the transaction isolation level, if specified.
  191. options = settings_dict['OPTIONS'].copy()
  192. isolation_level = options.pop('isolation_level', 'read committed')
  193. if isolation_level:
  194. isolation_level = isolation_level.lower()
  195. if isolation_level not in self.isolation_levels:
  196. raise ImproperlyConfigured(
  197. "Invalid transaction isolation level '%s' specified.\n"
  198. "Use one of %s, or None." % (
  199. isolation_level,
  200. ', '.join("'%s'" % s for s in sorted(self.isolation_levels))
  201. ))
  202. self.isolation_level = isolation_level
  203. kwargs.update(options)
  204. return kwargs
  205. @async_unsafe
  206. def get_new_connection(self, conn_params):
  207. return Database.connect(**conn_params)
  208. def init_connection_state(self):
  209. assignments = []
  210. if self.features.is_sql_auto_is_null_enabled:
  211. # SQL_AUTO_IS_NULL controls whether an AUTO_INCREMENT column on
  212. # a recently inserted row will return when the field is tested
  213. # for NULL. Disabling this brings this aspect of MySQL in line
  214. # with SQL standards.
  215. assignments.append('SET SQL_AUTO_IS_NULL = 0')
  216. if self.isolation_level:
  217. assignments.append('SET SESSION TRANSACTION ISOLATION LEVEL %s' % self.isolation_level.upper())
  218. if assignments:
  219. with self.cursor() as cursor:
  220. cursor.execute('; '.join(assignments))
  221. @async_unsafe
  222. def create_cursor(self, name=None):
  223. cursor = self.connection.cursor()
  224. return CursorWrapper(cursor)
  225. def _rollback(self):
  226. try:
  227. BaseDatabaseWrapper._rollback(self)
  228. except Database.NotSupportedError:
  229. pass
  230. def _set_autocommit(self, autocommit):
  231. with self.wrap_database_errors:
  232. self.connection.autocommit(autocommit)
  233. def disable_constraint_checking(self):
  234. """
  235. Disable foreign key checks, primarily for use in adding rows with
  236. forward references. Always return True to indicate constraint checks
  237. need to be re-enabled.
  238. """
  239. self.cursor().execute('SET foreign_key_checks=0')
  240. return True
  241. def enable_constraint_checking(self):
  242. """
  243. Re-enable foreign key checks after they have been disabled.
  244. """
  245. # Override needs_rollback in case constraint_checks_disabled is
  246. # nested inside transaction.atomic.
  247. self.needs_rollback, needs_rollback = False, self.needs_rollback
  248. try:
  249. self.cursor().execute('SET foreign_key_checks=1')
  250. finally:
  251. self.needs_rollback = needs_rollback
  252. def check_constraints(self, table_names=None):
  253. """
  254. Check each table name in `table_names` for rows with invalid foreign
  255. key references. This method is intended to be used in conjunction with
  256. `disable_constraint_checking()` and `enable_constraint_checking()`, to
  257. determine if rows with invalid references were entered while constraint
  258. checks were off.
  259. """
  260. with self.cursor() as cursor:
  261. if table_names is None:
  262. table_names = self.introspection.table_names(cursor)
  263. for table_name in table_names:
  264. primary_key_column_name = self.introspection.get_primary_key_column(cursor, table_name)
  265. if not primary_key_column_name:
  266. continue
  267. key_columns = self.introspection.get_key_columns(cursor, table_name)
  268. for column_name, referenced_table_name, referenced_column_name in key_columns:
  269. cursor.execute(
  270. """
  271. SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING
  272. LEFT JOIN `%s` as REFERRED
  273. ON (REFERRING.`%s` = REFERRED.`%s`)
  274. WHERE REFERRING.`%s` IS NOT NULL AND REFERRED.`%s` IS NULL
  275. """ % (
  276. primary_key_column_name, column_name, table_name,
  277. referenced_table_name, column_name, referenced_column_name,
  278. column_name, referenced_column_name,
  279. )
  280. )
  281. for bad_row in cursor.fetchall():
  282. raise utils.IntegrityError(
  283. "The row in table '%s' with primary key '%s' has an invalid "
  284. "foreign key: %s.%s contains a value '%s' that does not "
  285. "have a corresponding value in %s.%s."
  286. % (
  287. table_name, bad_row[0], table_name, column_name,
  288. bad_row[1], referenced_table_name, referenced_column_name,
  289. )
  290. )
  291. def is_usable(self):
  292. try:
  293. self.connection.ping()
  294. except Database.Error:
  295. return False
  296. else:
  297. return True
  298. @cached_property
  299. def display_name(self):
  300. return 'MariaDB' if self.mysql_is_mariadb else 'MySQL'
  301. @cached_property
  302. def data_type_check_constraints(self):
  303. if self.features.supports_column_check_constraints:
  304. return {
  305. 'PositiveIntegerField': '`%(column)s` >= 0',
  306. 'PositiveSmallIntegerField': '`%(column)s` >= 0',
  307. }
  308. return {}
  309. @cached_property
  310. def mysql_server_info(self):
  311. with self.temporary_connection() as cursor:
  312. cursor.execute('SELECT VERSION()')
  313. return cursor.fetchone()[0]
  314. @cached_property
  315. def mysql_version(self):
  316. match = server_version_re.match(self.mysql_server_info)
  317. if not match:
  318. raise Exception('Unable to determine MySQL version from version string %r' % self.mysql_server_info)
  319. return tuple(int(x) for x in match.groups())
  320. @cached_property
  321. def mysql_is_mariadb(self):
  322. return 'mariadb' in self.mysql_server_info.lower()