converters.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. from ._compat import PY2, text_type, long_type, JYTHON, IRONPYTHON, unichr
  2. import datetime
  3. from decimal import Decimal
  4. import re
  5. import time
  6. from .constants import FIELD_TYPE, FLAG
  7. from .charset import charset_by_id, charset_to_encoding
  8. def escape_item(val, charset, mapping=None):
  9. if mapping is None:
  10. mapping = encoders
  11. encoder = mapping.get(type(val))
  12. # Fallback to default when no encoder found
  13. if not encoder:
  14. try:
  15. encoder = mapping[text_type]
  16. except KeyError:
  17. raise TypeError("no default type converter defined")
  18. if encoder in (escape_dict, escape_sequence):
  19. val = encoder(val, charset, mapping)
  20. else:
  21. val = encoder(val, mapping)
  22. return val
  23. def escape_dict(val, charset, mapping=None):
  24. n = {}
  25. for k, v in val.items():
  26. quoted = escape_item(v, charset, mapping)
  27. n[k] = quoted
  28. return n
  29. def escape_sequence(val, charset, mapping=None):
  30. n = []
  31. for item in val:
  32. quoted = escape_item(item, charset, mapping)
  33. n.append(quoted)
  34. return "(" + ",".join(n) + ")"
  35. def escape_set(val, charset, mapping=None):
  36. return ','.join([escape_item(x, charset, mapping) for x in val])
  37. def escape_bool(value, mapping=None):
  38. return str(int(value))
  39. def escape_object(value, mapping=None):
  40. return str(value)
  41. def escape_int(value, mapping=None):
  42. return str(value)
  43. def escape_float(value, mapping=None):
  44. return ('%.15g' % value)
  45. _escape_table = [unichr(x) for x in range(128)]
  46. _escape_table[0] = u'\\0'
  47. _escape_table[ord('\\')] = u'\\\\'
  48. _escape_table[ord('\n')] = u'\\n'
  49. _escape_table[ord('\r')] = u'\\r'
  50. _escape_table[ord('\032')] = u'\\Z'
  51. _escape_table[ord('"')] = u'\\"'
  52. _escape_table[ord("'")] = u"\\'"
  53. def _escape_unicode(value, mapping=None):
  54. """escapes *value* without adding quote.
  55. Value should be unicode
  56. """
  57. return value.translate(_escape_table)
  58. if PY2:
  59. def escape_string(value, mapping=None):
  60. """escape_string escapes *value* but not surround it with quotes.
  61. Value should be bytes or unicode.
  62. """
  63. if isinstance(value, unicode):
  64. return _escape_unicode(value)
  65. assert isinstance(value, (bytes, bytearray))
  66. value = value.replace('\\', '\\\\')
  67. value = value.replace('\0', '\\0')
  68. value = value.replace('\n', '\\n')
  69. value = value.replace('\r', '\\r')
  70. value = value.replace('\032', '\\Z')
  71. value = value.replace("'", "\\'")
  72. value = value.replace('"', '\\"')
  73. return value
  74. def escape_bytes_prefixed(value, mapping=None):
  75. assert isinstance(value, (bytes, bytearray))
  76. return b"_binary'%s'" % escape_string(value)
  77. def escape_bytes(value, mapping=None):
  78. assert isinstance(value, (bytes, bytearray))
  79. return b"'%s'" % escape_string(value)
  80. else:
  81. escape_string = _escape_unicode
  82. # On Python ~3.5, str.decode('ascii', 'surrogateescape') is slow.
  83. # (fixed in Python 3.6, http://bugs.python.org/issue24870)
  84. # Workaround is str.decode('latin1') then translate 0x80-0xff into 0udc80-0udcff.
  85. # We can escape special chars and surrogateescape at once.
  86. _escape_bytes_table = _escape_table + [chr(i) for i in range(0xdc80, 0xdd00)]
  87. def escape_bytes_prefixed(value, mapping=None):
  88. return "_binary'%s'" % value.decode('latin1').translate(_escape_bytes_table)
  89. def escape_bytes(value, mapping=None):
  90. return "'%s'" % value.decode('latin1').translate(_escape_bytes_table)
  91. def escape_unicode(value, mapping=None):
  92. return u"'%s'" % _escape_unicode(value)
  93. def escape_str(value, mapping=None):
  94. return "'%s'" % escape_string(str(value), mapping)
  95. def escape_None(value, mapping=None):
  96. return 'NULL'
  97. def escape_timedelta(obj, mapping=None):
  98. seconds = int(obj.seconds) % 60
  99. minutes = int(obj.seconds // 60) % 60
  100. hours = int(obj.seconds // 3600) % 24 + int(obj.days) * 24
  101. if obj.microseconds:
  102. fmt = "'{0:02d}:{1:02d}:{2:02d}.{3:06d}'"
  103. else:
  104. fmt = "'{0:02d}:{1:02d}:{2:02d}'"
  105. return fmt.format(hours, minutes, seconds, obj.microseconds)
  106. def escape_time(obj, mapping=None):
  107. if obj.microsecond:
  108. fmt = "'{0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'"
  109. else:
  110. fmt = "'{0.hour:02}:{0.minute:02}:{0.second:02}'"
  111. return fmt.format(obj)
  112. def escape_datetime(obj, mapping=None):
  113. if obj.microsecond:
  114. fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'"
  115. else:
  116. fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}'"
  117. return fmt.format(obj)
  118. def escape_date(obj, mapping=None):
  119. fmt = "'{0.year:04}-{0.month:02}-{0.day:02}'"
  120. return fmt.format(obj)
  121. def escape_struct_time(obj, mapping=None):
  122. return escape_datetime(datetime.datetime(*obj[:6]))
  123. def _convert_second_fraction(s):
  124. if not s:
  125. return 0
  126. # Pad zeros to ensure the fraction length in microseconds
  127. s = s.ljust(6, '0')
  128. return int(s[:6])
  129. DATETIME_RE = re.compile(r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
  130. def convert_datetime(obj):
  131. """Returns a DATETIME or TIMESTAMP column value as a datetime object:
  132. >>> datetime_or_None('2007-02-25 23:06:20')
  133. datetime.datetime(2007, 2, 25, 23, 6, 20)
  134. >>> datetime_or_None('2007-02-25T23:06:20')
  135. datetime.datetime(2007, 2, 25, 23, 6, 20)
  136. Illegal values are returned as None:
  137. >>> datetime_or_None('2007-02-31T23:06:20') is None
  138. True
  139. >>> datetime_or_None('0000-00-00 00:00:00') is None
  140. True
  141. """
  142. if not PY2 and isinstance(obj, (bytes, bytearray)):
  143. obj = obj.decode('ascii')
  144. m = DATETIME_RE.match(obj)
  145. if not m:
  146. return convert_date(obj)
  147. try:
  148. groups = list(m.groups())
  149. groups[-1] = _convert_second_fraction(groups[-1])
  150. return datetime.datetime(*[ int(x) for x in groups ])
  151. except ValueError:
  152. return convert_date(obj)
  153. TIMEDELTA_RE = re.compile(r"(-)?(\d{1,3}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
  154. def convert_timedelta(obj):
  155. """Returns a TIME column as a timedelta object:
  156. >>> timedelta_or_None('25:06:17')
  157. datetime.timedelta(1, 3977)
  158. >>> timedelta_or_None('-25:06:17')
  159. datetime.timedelta(-2, 83177)
  160. Illegal values are returned as None:
  161. >>> timedelta_or_None('random crap') is None
  162. True
  163. Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but
  164. can accept values as (+|-)DD HH:MM:SS. The latter format will not
  165. be parsed correctly by this function.
  166. """
  167. if not PY2 and isinstance(obj, (bytes, bytearray)):
  168. obj = obj.decode('ascii')
  169. m = TIMEDELTA_RE.match(obj)
  170. if not m:
  171. return obj
  172. try:
  173. groups = list(m.groups())
  174. groups[-1] = _convert_second_fraction(groups[-1])
  175. negate = -1 if groups[0] else 1
  176. hours, minutes, seconds, microseconds = groups[1:]
  177. tdelta = datetime.timedelta(
  178. hours = int(hours),
  179. minutes = int(minutes),
  180. seconds = int(seconds),
  181. microseconds = int(microseconds)
  182. ) * negate
  183. return tdelta
  184. except ValueError:
  185. return obj
  186. TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
  187. def convert_time(obj):
  188. """Returns a TIME column as a time object:
  189. >>> time_or_None('15:06:17')
  190. datetime.time(15, 6, 17)
  191. Illegal values are returned as None:
  192. >>> time_or_None('-25:06:17') is None
  193. True
  194. >>> time_or_None('random crap') is None
  195. True
  196. Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but
  197. can accept values as (+|-)DD HH:MM:SS. The latter format will not
  198. be parsed correctly by this function.
  199. Also note that MySQL's TIME column corresponds more closely to
  200. Python's timedelta and not time. However if you want TIME columns
  201. to be treated as time-of-day and not a time offset, then you can
  202. use set this function as the converter for FIELD_TYPE.TIME.
  203. """
  204. if not PY2 and isinstance(obj, (bytes, bytearray)):
  205. obj = obj.decode('ascii')
  206. m = TIME_RE.match(obj)
  207. if not m:
  208. return obj
  209. try:
  210. groups = list(m.groups())
  211. groups[-1] = _convert_second_fraction(groups[-1])
  212. hours, minutes, seconds, microseconds = groups
  213. return datetime.time(hour=int(hours), minute=int(minutes),
  214. second=int(seconds), microsecond=int(microseconds))
  215. except ValueError:
  216. return obj
  217. def convert_date(obj):
  218. """Returns a DATE column as a date object:
  219. >>> date_or_None('2007-02-26')
  220. datetime.date(2007, 2, 26)
  221. Illegal values are returned as None:
  222. >>> date_or_None('2007-02-31') is None
  223. True
  224. >>> date_or_None('0000-00-00') is None
  225. True
  226. """
  227. if not PY2 and isinstance(obj, (bytes, bytearray)):
  228. obj = obj.decode('ascii')
  229. try:
  230. return datetime.date(*[ int(x) for x in obj.split('-', 2) ])
  231. except ValueError:
  232. return obj
  233. def convert_mysql_timestamp(timestamp):
  234. """Convert a MySQL TIMESTAMP to a Timestamp object.
  235. MySQL >= 4.1 returns TIMESTAMP in the same format as DATETIME:
  236. >>> mysql_timestamp_converter('2007-02-25 22:32:17')
  237. datetime.datetime(2007, 2, 25, 22, 32, 17)
  238. MySQL < 4.1 uses a big string of numbers:
  239. >>> mysql_timestamp_converter('20070225223217')
  240. datetime.datetime(2007, 2, 25, 22, 32, 17)
  241. Illegal values are returned as None:
  242. >>> mysql_timestamp_converter('2007-02-31 22:32:17') is None
  243. True
  244. >>> mysql_timestamp_converter('00000000000000') is None
  245. True
  246. """
  247. if not PY2 and isinstance(timestamp, (bytes, bytearray)):
  248. timestamp = timestamp.decode('ascii')
  249. if timestamp[4] == '-':
  250. return convert_datetime(timestamp)
  251. timestamp += "0"*(14-len(timestamp)) # padding
  252. year, month, day, hour, minute, second = \
  253. int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8]), \
  254. int(timestamp[8:10]), int(timestamp[10:12]), int(timestamp[12:14])
  255. try:
  256. return datetime.datetime(year, month, day, hour, minute, second)
  257. except ValueError:
  258. return timestamp
  259. def convert_set(s):
  260. if isinstance(s, (bytes, bytearray)):
  261. return set(s.split(b","))
  262. return set(s.split(","))
  263. def through(x):
  264. return x
  265. #def convert_bit(b):
  266. # b = "\x00" * (8 - len(b)) + b # pad w/ zeroes
  267. # return struct.unpack(">Q", b)[0]
  268. #
  269. # the snippet above is right, but MySQLdb doesn't process bits,
  270. # so we shouldn't either
  271. convert_bit = through
  272. encoders = {
  273. bool: escape_bool,
  274. int: escape_int,
  275. long_type: escape_int,
  276. float: escape_float,
  277. str: escape_str,
  278. text_type: escape_unicode,
  279. tuple: escape_sequence,
  280. list: escape_sequence,
  281. set: escape_sequence,
  282. frozenset: escape_sequence,
  283. dict: escape_dict,
  284. type(None): escape_None,
  285. datetime.date: escape_date,
  286. datetime.datetime: escape_datetime,
  287. datetime.timedelta: escape_timedelta,
  288. datetime.time: escape_time,
  289. time.struct_time: escape_struct_time,
  290. Decimal: escape_object,
  291. }
  292. if not PY2 or JYTHON or IRONPYTHON:
  293. encoders[bytes] = escape_bytes
  294. decoders = {
  295. FIELD_TYPE.BIT: convert_bit,
  296. FIELD_TYPE.TINY: int,
  297. FIELD_TYPE.SHORT: int,
  298. FIELD_TYPE.LONG: int,
  299. FIELD_TYPE.FLOAT: float,
  300. FIELD_TYPE.DOUBLE: float,
  301. FIELD_TYPE.LONGLONG: int,
  302. FIELD_TYPE.INT24: int,
  303. FIELD_TYPE.YEAR: int,
  304. FIELD_TYPE.TIMESTAMP: convert_mysql_timestamp,
  305. FIELD_TYPE.DATETIME: convert_datetime,
  306. FIELD_TYPE.TIME: convert_timedelta,
  307. FIELD_TYPE.DATE: convert_date,
  308. FIELD_TYPE.SET: convert_set,
  309. FIELD_TYPE.BLOB: through,
  310. FIELD_TYPE.TINY_BLOB: through,
  311. FIELD_TYPE.MEDIUM_BLOB: through,
  312. FIELD_TYPE.LONG_BLOB: through,
  313. FIELD_TYPE.STRING: through,
  314. FIELD_TYPE.VAR_STRING: through,
  315. FIELD_TYPE.VARCHAR: through,
  316. FIELD_TYPE.DECIMAL: Decimal,
  317. FIELD_TYPE.NEWDECIMAL: Decimal,
  318. }
  319. # for MySQLdb compatibility
  320. conversions = encoders.copy()
  321. conversions.update(decoders)
  322. Thing2Literal = escape_str