/arch/x86/crypto/aesni-intel_asm.S

https://bitbucket.org/cresqo/cm7-p500-kernel · Assembly · 841 lines · 682 code · 27 blank · 132 comment · 0 complexity · f43d6821db153f6aacc65bacc4295382 MD5 · raw file 

    

  1. /*
    2. 

  2.  * Implement AES algorithm in Intel AES-NI instructions.
    3. 

  3.  *
    4. 

  4.  * The white paper of AES-NI instructions can be downloaded from:
    5. 

  5.  *   http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
    6. 

  6.  *
    7. 

  7.  * Copyright (C) 2008, Intel Corp.
    8. 

  8.  *    Author: Huang Ying <ying.huang@intel.com>
    9. 

  9.  *            Vinodh Gopal <vinodh.gopal@intel.com>
    10. 

  10.  *            Kahraman Akdemir
    11. 

  11.  *
    12. 

  12.  * This program is free software; you can redistribute it and/or modify
    13. 

  13.  * it under the terms of the GNU General Public License as published by
    14. 

  14.  * the Free Software Foundation; either version 2 of the License, or
    15. 

  15.  * (at your option) any later version.
    16. 

  16.  */
    17. 

  17. 

  18. #include <linux/linkage.h>
    19. 

  19. #include <asm/inst.h>
    20. 

  20. 

  21. .text
    22. 

  22. 

  23. #define STATE1	%xmm0
    24. 

  24. #define STATE2	%xmm4
    25. 

  25. #define STATE3	%xmm5
    26. 

  26. #define STATE4	%xmm6
    27. 

  27. #define STATE	STATE1
    28. 

  28. #define IN1	%xmm1
    29. 

  29. #define IN2	%xmm7
    30. 

  30. #define IN3	%xmm8
    31. 

  31. #define IN4	%xmm9
    32. 

  32. #define IN	IN1
    33. 

  33. #define KEY	%xmm2
    34. 

  34. #define IV	%xmm3
    35. 

  35. #define BSWAP_MASK %xmm10
    36. 

  36. #define CTR	%xmm11
    37. 

  37. #define INC	%xmm12
    38. 

  38. 

  39. #define KEYP	%rdi
    40. 

  40. #define OUTP	%rsi
    41. 

  41. #define INP	%rdx
    42. 

  42. #define LEN	%rcx
    43. 

  43. #define IVP	%r8
    44. 

  44. #define KLEN	%r9d
    45. 

  45. #define T1	%r10
    46. 

  46. #define TKEYP	T1
    47. 

  47. #define T2	%r11
    48. 

  48. #define TCTR_LOW T2
    49. 

  49. 

  50. _key_expansion_128:
    51. 

  51. _key_expansion_256a:
    52. 

  52. 	pshufd $0b11111111, %xmm1, %xmm1
    53. 

  53. 	shufps $0b00010000, %xmm0, %xmm4
    54. 

  54. 	pxor %xmm4, %xmm0
    55. 

  55. 	shufps $0b10001100, %xmm0, %xmm4
    56. 

  56. 	pxor %xmm4, %xmm0
    57. 

  57. 	pxor %xmm1, %xmm0
    58. 

  58. 	movaps %xmm0, (%rcx)
    59. 

  59. 	add $0x10, %rcx
    60. 

  60. 	ret
    61. 

  61. 

  62. _key_expansion_192a:
    63. 

  63. 	pshufd $0b01010101, %xmm1, %xmm1
    64. 

  64. 	shufps $0b00010000, %xmm0, %xmm4
    65. 

  65. 	pxor %xmm4, %xmm0
    66. 

  66. 	shufps $0b10001100, %xmm0, %xmm4
    67. 

  67. 	pxor %xmm4, %xmm0
    68. 

  68. 	pxor %xmm1, %xmm0
    69. 

  69. 

  70. 	movaps %xmm2, %xmm5
    71. 

  71. 	movaps %xmm2, %xmm6
    72. 

  72. 	pslldq $4, %xmm5
    73. 

  73. 	pshufd $0b11111111, %xmm0, %xmm3
    74. 

  74. 	pxor %xmm3, %xmm2
    75. 

  75. 	pxor %xmm5, %xmm2
    76. 

  76. 

  77. 	movaps %xmm0, %xmm1
    78. 

  78. 	shufps $0b01000100, %xmm0, %xmm6
    79. 

  79. 	movaps %xmm6, (%rcx)
    80. 

  80. 	shufps $0b01001110, %xmm2, %xmm1
    81. 

  81. 	movaps %xmm1, 16(%rcx)
    82. 

  82. 	add $0x20, %rcx
    83. 

  83. 	ret
    84. 

  84. 

  85. _key_expansion_192b:
    86. 

  86. 	pshufd $0b01010101, %xmm1, %xmm1
    87. 

  87. 	shufps $0b00010000, %xmm0, %xmm4
    88. 

  88. 	pxor %xmm4, %xmm0
    89. 

  89. 	shufps $0b10001100, %xmm0, %xmm4
    90. 

  90. 	pxor %xmm4, %xmm0
    91. 

  91. 	pxor %xmm1, %xmm0
    92. 

  92. 

  93. 	movaps %xmm2, %xmm5
    94. 

  94. 	pslldq $4, %xmm5
    95. 

  95. 	pshufd $0b11111111, %xmm0, %xmm3
    96. 

  96. 	pxor %xmm3, %xmm2
    97. 

  97. 	pxor %xmm5, %xmm2
    98. 

  98. 

  99. 	movaps %xmm0, (%rcx)
    100. 

  100. 	add $0x10, %rcx
    101. 

  101. 	ret
    102. 

  102. 

  103. _key_expansion_256b:
    104. 

  104. 	pshufd $0b10101010, %xmm1, %xmm1
    105. 

  105. 	shufps $0b00010000, %xmm2, %xmm4
    106. 

  106. 	pxor %xmm4, %xmm2
    107. 

  107. 	shufps $0b10001100, %xmm2, %xmm4
    108. 

  108. 	pxor %xmm4, %xmm2
    109. 

  109. 	pxor %xmm1, %xmm2
    110. 

  110. 	movaps %xmm2, (%rcx)
    111. 

  111. 	add $0x10, %rcx
    112. 

  112. 	ret
    113. 

  113. 

  114. /*
    115. 

  115.  * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
    116. 

  116.  *                   unsigned int key_len)
    117. 

  117.  */
    118. 

  118. ENTRY(aesni_set_key)
    119. 

  119. 	movups (%rsi), %xmm0		# user key (first 16 bytes)
    120. 

  120. 	movaps %xmm0, (%rdi)
    121. 

  121. 	lea 0x10(%rdi), %rcx		# key addr
    122. 

  122. 	movl %edx, 480(%rdi)
    123. 

  123. 	pxor %xmm4, %xmm4		# xmm4 is assumed 0 in _key_expansion_x
    124. 

  124. 	cmp $24, %dl
    125. 

  125. 	jb .Lenc_key128
    126. 

  126. 	je .Lenc_key192
    127. 

  127. 	movups 0x10(%rsi), %xmm2	# other user key
    128. 

  128. 	movaps %xmm2, (%rcx)
    129. 

  129. 	add $0x10, %rcx
    130. 

  130. 	AESKEYGENASSIST 0x1 %xmm2 %xmm1		# round 1
    131. 

  131. 	call _key_expansion_256a
    132. 

  132. 	AESKEYGENASSIST 0x1 %xmm0 %xmm1
    133. 

  133. 	call _key_expansion_256b
    134. 

  134. 	AESKEYGENASSIST 0x2 %xmm2 %xmm1		# round 2
    135. 

  135. 	call _key_expansion_256a
    136. 

  136. 	AESKEYGENASSIST 0x2 %xmm0 %xmm1
    137. 

  137. 	call _key_expansion_256b
    138. 

  138. 	AESKEYGENASSIST 0x4 %xmm2 %xmm1		# round 3
    139. 

  139. 	call _key_expansion_256a
    140. 

  140. 	AESKEYGENASSIST 0x4 %xmm0 %xmm1
    141. 

  141. 	call _key_expansion_256b
    142. 

  142. 	AESKEYGENASSIST 0x8 %xmm2 %xmm1		# round 4
    143. 

  143. 	call _key_expansion_256a
    144. 

  144. 	AESKEYGENASSIST 0x8 %xmm0 %xmm1
    145. 

  145. 	call _key_expansion_256b
    146. 

  146. 	AESKEYGENASSIST 0x10 %xmm2 %xmm1	# round 5
    147. 

  147. 	call _key_expansion_256a
    148. 

  148. 	AESKEYGENASSIST 0x10 %xmm0 %xmm1
    149. 

  149. 	call _key_expansion_256b
    150. 

  150. 	AESKEYGENASSIST 0x20 %xmm2 %xmm1	# round 6
    151. 

  151. 	call _key_expansion_256a
    152. 

  152. 	AESKEYGENASSIST 0x20 %xmm0 %xmm1
    153. 

  153. 	call _key_expansion_256b
    154. 

  154. 	AESKEYGENASSIST 0x40 %xmm2 %xmm1	# round 7
    155. 

  155. 	call _key_expansion_256a
    156. 

  156. 	jmp .Ldec_key
    157. 

  157. .Lenc_key192:
    158. 

  158. 	movq 0x10(%rsi), %xmm2		# other user key
    159. 

  159. 	AESKEYGENASSIST 0x1 %xmm2 %xmm1		# round 1
    160. 

  160. 	call _key_expansion_192a
    161. 

  161. 	AESKEYGENASSIST 0x2 %xmm2 %xmm1		# round 2
    162. 

  162. 	call _key_expansion_192b
    163. 

  163. 	AESKEYGENASSIST 0x4 %xmm2 %xmm1		# round 3
    164. 

  164. 	call _key_expansion_192a
    165. 

  165. 	AESKEYGENASSIST 0x8 %xmm2 %xmm1		# round 4
    166. 

  166. 	call _key_expansion_192b
    167. 

  167. 	AESKEYGENASSIST 0x10 %xmm2 %xmm1	# round 5
    168. 

  168. 	call _key_expansion_192a
    169. 

  169. 	AESKEYGENASSIST 0x20 %xmm2 %xmm1	# round 6
    170. 

  170. 	call _key_expansion_192b
    171. 

  171. 	AESKEYGENASSIST 0x40 %xmm2 %xmm1	# round 7
    172. 

  172. 	call _key_expansion_192a
    173. 

  173. 	AESKEYGENASSIST 0x80 %xmm2 %xmm1	# round 8
    174. 

  174. 	call _key_expansion_192b
    175. 

  175. 	jmp .Ldec_key
    176. 

  176. .Lenc_key128:
    177. 

  177. 	AESKEYGENASSIST 0x1 %xmm0 %xmm1		# round 1
    178. 

  178. 	call _key_expansion_128
    179. 

  179. 	AESKEYGENASSIST 0x2 %xmm0 %xmm1		# round 2
    180. 

  180. 	call _key_expansion_128
    181. 

  181. 	AESKEYGENASSIST 0x4 %xmm0 %xmm1		# round 3
    182. 

  182. 	call _key_expansion_128
    183. 

  183. 	AESKEYGENASSIST 0x8 %xmm0 %xmm1		# round 4
    184. 

  184. 	call _key_expansion_128
    185. 

  185. 	AESKEYGENASSIST 0x10 %xmm0 %xmm1	# round 5
    186. 

  186. 	call _key_expansion_128
    187. 

  187. 	AESKEYGENASSIST 0x20 %xmm0 %xmm1	# round 6
    188. 

  188. 	call _key_expansion_128
    189. 

  189. 	AESKEYGENASSIST 0x40 %xmm0 %xmm1	# round 7
    190. 

  190. 	call _key_expansion_128
    191. 

  191. 	AESKEYGENASSIST 0x80 %xmm0 %xmm1	# round 8
    192. 

  192. 	call _key_expansion_128
    193. 

  193. 	AESKEYGENASSIST 0x1b %xmm0 %xmm1	# round 9
    194. 

  194. 	call _key_expansion_128
    195. 

  195. 	AESKEYGENASSIST 0x36 %xmm0 %xmm1	# round 10
    196. 

  196. 	call _key_expansion_128
    197. 

  197. .Ldec_key:
    198. 

  198. 	sub $0x10, %rcx
    199. 

  199. 	movaps (%rdi), %xmm0
    200. 

  200. 	movaps (%rcx), %xmm1
    201. 

  201. 	movaps %xmm0, 240(%rcx)
    202. 

  202. 	movaps %xmm1, 240(%rdi)
    203. 

  203. 	add $0x10, %rdi
    204. 

  204. 	lea 240-16(%rcx), %rsi
    205. 

  205. .align 4
    206. 

  206. .Ldec_key_loop:
    207. 

  207. 	movaps (%rdi), %xmm0
    208. 

  208. 	AESIMC %xmm0 %xmm1
    209. 

  209. 	movaps %xmm1, (%rsi)
    210. 

  210. 	add $0x10, %rdi
    211. 

  211. 	sub $0x10, %rsi
    212. 

  212. 	cmp %rcx, %rdi
    213. 

  213. 	jb .Ldec_key_loop
    214. 

  214. 	xor %rax, %rax
    215. 

  215. 	ret
    216. 

  216. 

  217. /*
    218. 

  218.  * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
    219. 

  219.  */
    220. 

  220. ENTRY(aesni_enc)
    221. 

  221. 	movl 480(KEYP), KLEN		# key length
    222. 

  222. 	movups (INP), STATE		# input
    223. 

  223. 	call _aesni_enc1
    224. 

  224. 	movups STATE, (OUTP)		# output
    225. 

  225. 	ret
    226. 

  226. 

  227. /*
    228. 

  228.  * _aesni_enc1:		internal ABI
    229. 

  229.  * input:
    230. 

  230.  *	KEYP:		key struct pointer
    231. 

  231.  *	KLEN:		round count
    232. 

  232.  *	STATE:		initial state (input)
    233. 

  233.  * output:
    234. 

  234.  *	STATE:		finial state (output)
    235. 

  235.  * changed:
    236. 

  236.  *	KEY
    237. 

  237.  *	TKEYP (T1)
    238. 

  238.  */
    239. 

  239. _aesni_enc1:
    240. 

  240. 	movaps (KEYP), KEY		# key
    241. 

  241. 	mov KEYP, TKEYP
    242. 

  242. 	pxor KEY, STATE		# round 0
    243. 

  243. 	add $0x30, TKEYP
    244. 

  244. 	cmp $24, KLEN
    245. 

  245. 	jb .Lenc128
    246. 

  246. 	lea 0x20(TKEYP), TKEYP
    247. 

  247. 	je .Lenc192
    248. 

  248. 	add $0x20, TKEYP
    249. 

  249. 	movaps -0x60(TKEYP), KEY
    250. 

  250. 	AESENC KEY STATE
    251. 

  251. 	movaps -0x50(TKEYP), KEY
    252. 

  252. 	AESENC KEY STATE
    253. 

  253. .align 4
    254. 

  254. .Lenc192:
    255. 

  255. 	movaps -0x40(TKEYP), KEY
    256. 

  256. 	AESENC KEY STATE
    257. 

  257. 	movaps -0x30(TKEYP), KEY
    258. 

  258. 	AESENC KEY STATE
    259. 

  259. .align 4
    260. 

  260. .Lenc128:
    261. 

  261. 	movaps -0x20(TKEYP), KEY
    262. 

  262. 	AESENC KEY STATE
    263. 

  263. 	movaps -0x10(TKEYP), KEY
    264. 

  264. 	AESENC KEY STATE
    265. 

  265. 	movaps (TKEYP), KEY
    266. 

  266. 	AESENC KEY STATE
    267. 

  267. 	movaps 0x10(TKEYP), KEY
    268. 

  268. 	AESENC KEY STATE
    269. 

  269. 	movaps 0x20(TKEYP), KEY
    270. 

  270. 	AESENC KEY STATE
    271. 

  271. 	movaps 0x30(TKEYP), KEY
    272. 

  272. 	AESENC KEY STATE
    273. 

  273. 	movaps 0x40(TKEYP), KEY
    274. 

  274. 	AESENC KEY STATE
    275. 

  275. 	movaps 0x50(TKEYP), KEY
    276. 

  276. 	AESENC KEY STATE
    277. 

  277. 	movaps 0x60(TKEYP), KEY
    278. 

  278. 	AESENC KEY STATE
    279. 

  279. 	movaps 0x70(TKEYP), KEY
    280. 

  280. 	AESENCLAST KEY STATE
    281. 

  281. 	ret
    282. 

  282. 

  283. /*
    284. 

  284.  * _aesni_enc4:	internal ABI
    285. 

  285.  * input:
    286. 

  286.  *	KEYP:		key struct pointer
    287. 

  287.  *	KLEN:		round count
    288. 

  288.  *	STATE1:		initial state (input)
    289. 

  289.  *	STATE2
    290. 

  290.  *	STATE3
    291. 

  291.  *	STATE4
    292. 

  292.  * output:
    293. 

  293.  *	STATE1:		finial state (output)
    294. 

  294.  *	STATE2
    295. 

  295.  *	STATE3
    296. 

  296.  *	STATE4
    297. 

  297.  * changed:
    298. 

  298.  *	KEY
    299. 

  299.  *	TKEYP (T1)
    300. 

  300.  */
    301. 

  301. _aesni_enc4:
    302. 

  302. 	movaps (KEYP), KEY		# key
    303. 

  303. 	mov KEYP, TKEYP
    304. 

  304. 	pxor KEY, STATE1		# round 0
    305. 

  305. 	pxor KEY, STATE2
    306. 

  306. 	pxor KEY, STATE3
    307. 

  307. 	pxor KEY, STATE4
    308. 

  308. 	add $0x30, TKEYP
    309. 

  309. 	cmp $24, KLEN
    310. 

  310. 	jb .L4enc128
    311. 

  311. 	lea 0x20(TKEYP), TKEYP
    312. 

  312. 	je .L4enc192
    313. 

  313. 	add $0x20, TKEYP
    314. 

  314. 	movaps -0x60(TKEYP), KEY
    315. 

  315. 	AESENC KEY STATE1
    316. 

  316. 	AESENC KEY STATE2
    317. 

  317. 	AESENC KEY STATE3
    318. 

  318. 	AESENC KEY STATE4
    319. 

  319. 	movaps -0x50(TKEYP), KEY
    320. 

  320. 	AESENC KEY STATE1
    321. 

  321. 	AESENC KEY STATE2
    322. 

  322. 	AESENC KEY STATE3
    323. 

  323. 	AESENC KEY STATE4
    324. 

  324. #.align 4
    325. 

  325. .L4enc192:
    326. 

  326. 	movaps -0x40(TKEYP), KEY
    327. 

  327. 	AESENC KEY STATE1
    328. 

  328. 	AESENC KEY STATE2
    329. 

  329. 	AESENC KEY STATE3
    330. 

  330. 	AESENC KEY STATE4
    331. 

  331. 	movaps -0x30(TKEYP), KEY
    332. 

  332. 	AESENC KEY STATE1
    333. 

  333. 	AESENC KEY STATE2
    334. 

  334. 	AESENC KEY STATE3
    335. 

  335. 	AESENC KEY STATE4
    336. 

  336. #.align 4
    337. 

  337. .L4enc128:
    338. 

  338. 	movaps -0x20(TKEYP), KEY
    339. 

  339. 	AESENC KEY STATE1
    340. 

  340. 	AESENC KEY STATE2
    341. 

  341. 	AESENC KEY STATE3
    342. 

  342. 	AESENC KEY STATE4
    343. 

  343. 	movaps -0x10(TKEYP), KEY
    344. 

  344. 	AESENC KEY STATE1
    345. 

  345. 	AESENC KEY STATE2
    346. 

  346. 	AESENC KEY STATE3
    347. 

  347. 	AESENC KEY STATE4
    348. 

  348. 	movaps (TKEYP), KEY
    349. 

  349. 	AESENC KEY STATE1
    350. 

  350. 	AESENC KEY STATE2
    351. 

  351. 	AESENC KEY STATE3
    352. 

  352. 	AESENC KEY STATE4
    353. 

  353. 	movaps 0x10(TKEYP), KEY
    354. 

  354. 	AESENC KEY STATE1
    355. 

  355. 	AESENC KEY STATE2
    356. 

  356. 	AESENC KEY STATE3
    357. 

  357. 	AESENC KEY STATE4
    358. 

  358. 	movaps 0x20(TKEYP), KEY
    359. 

  359. 	AESENC KEY STATE1
    360. 

  360. 	AESENC KEY STATE2
    361. 

  361. 	AESENC KEY STATE3
    362. 

  362. 	AESENC KEY STATE4
    363. 

  363. 	movaps 0x30(TKEYP), KEY
    364. 

  364. 	AESENC KEY STATE1
    365. 

  365. 	AESENC KEY STATE2
    366. 

  366. 	AESENC KEY STATE3
    367. 

  367. 	AESENC KEY STATE4
    368. 

  368. 	movaps 0x40(TKEYP), KEY
    369. 

  369. 	AESENC KEY STATE1
    370. 

  370. 	AESENC KEY STATE2
    371. 

  371. 	AESENC KEY STATE3
    372. 

  372. 	AESENC KEY STATE4
    373. 

  373. 	movaps 0x50(TKEYP), KEY
    374. 

  374. 	AESENC KEY STATE1
    375. 

  375. 	AESENC KEY STATE2
    376. 

  376. 	AESENC KEY STATE3
    377. 

  377. 	AESENC KEY STATE4
    378. 

  378. 	movaps 0x60(TKEYP), KEY
    379. 

  379. 	AESENC KEY STATE1
    380. 

  380. 	AESENC KEY STATE2
    381. 

  381. 	AESENC KEY STATE3
    382. 

  382. 	AESENC KEY STATE4
    383. 

  383. 	movaps 0x70(TKEYP), KEY
    384. 

  384. 	AESENCLAST KEY STATE1		# last round
    385. 

  385. 	AESENCLAST KEY STATE2
    386. 

  386. 	AESENCLAST KEY STATE3
    387. 

  387. 	AESENCLAST KEY STATE4
    388. 

  388. 	ret
    389. 

  389. 

  390. /*
    391. 

  391.  * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
    392. 

  392.  */
    393. 

  393. ENTRY(aesni_dec)
    394. 

  394. 	mov 480(KEYP), KLEN		# key length
    395. 

  395. 	add $240, KEYP
    396. 

  396. 	movups (INP), STATE		# input
    397. 

  397. 	call _aesni_dec1
    398. 

  398. 	movups STATE, (OUTP)		#output
    399. 

  399. 	ret
    400. 

  400. 

  401. /*
    402. 

  402.  * _aesni_dec1:		internal ABI
    403. 

  403.  * input:
    404. 

  404.  *	KEYP:		key struct pointer
    405. 

  405.  *	KLEN:		key length
    406. 

  406.  *	STATE:		initial state (input)
    407. 

  407.  * output:
    408. 

  408.  *	STATE:		finial state (output)
    409. 

  409.  * changed:
    410. 

  410.  *	KEY
    411. 

  411.  *	TKEYP (T1)
    412. 

  412.  */
    413. 

  413. _aesni_dec1:
    414. 

  414. 	movaps (KEYP), KEY		# key
    415. 

  415. 	mov KEYP, TKEYP
    416. 

  416. 	pxor KEY, STATE		# round 0
    417. 

  417. 	add $0x30, TKEYP
    418. 

  418. 	cmp $24, KLEN
    419. 

  419. 	jb .Ldec128
    420. 

  420. 	lea 0x20(TKEYP), TKEYP
    421. 

  421. 	je .Ldec192
    422. 

  422. 	add $0x20, TKEYP
    423. 

  423. 	movaps -0x60(TKEYP), KEY
    424. 

  424. 	AESDEC KEY STATE
    425. 

  425. 	movaps -0x50(TKEYP), KEY
    426. 

  426. 	AESDEC KEY STATE
    427. 

  427. .align 4
    428. 

  428. .Ldec192:
    429. 

  429. 	movaps -0x40(TKEYP), KEY
    430. 

  430. 	AESDEC KEY STATE
    431. 

  431. 	movaps -0x30(TKEYP), KEY
    432. 

  432. 	AESDEC KEY STATE
    433. 

  433. .align 4
    434. 

  434. .Ldec128:
    435. 

  435. 	movaps -0x20(TKEYP), KEY
    436. 

  436. 	AESDEC KEY STATE
    437. 

  437. 	movaps -0x10(TKEYP), KEY
    438. 

  438. 	AESDEC KEY STATE
    439. 

  439. 	movaps (TKEYP), KEY
    440. 

  440. 	AESDEC KEY STATE
    441. 

  441. 	movaps 0x10(TKEYP), KEY
    442. 

  442. 	AESDEC KEY STATE
    443. 

  443. 	movaps 0x20(TKEYP), KEY
    444. 

  444. 	AESDEC KEY STATE
    445. 

  445. 	movaps 0x30(TKEYP), KEY
    446. 

  446. 	AESDEC KEY STATE
    447. 

  447. 	movaps 0x40(TKEYP), KEY
    448. 

  448. 	AESDEC KEY STATE
    449. 

  449. 	movaps 0x50(TKEYP), KEY
    450. 

  450. 	AESDEC KEY STATE
    451. 

  451. 	movaps 0x60(TKEYP), KEY
    452. 

  452. 	AESDEC KEY STATE
    453. 

  453. 	movaps 0x70(TKEYP), KEY
    454. 

  454. 	AESDECLAST KEY STATE
    455. 

  455. 	ret
    456. 

  456. 

  457. /*
    458. 

  458.  * _aesni_dec4:	internal ABI
    459. 

  459.  * input:
    460. 

  460.  *	KEYP:		key struct pointer
    461. 

  461.  *	KLEN:		key length
    462. 

  462.  *	STATE1:		initial state (input)
    463. 

  463.  *	STATE2
    464. 

  464.  *	STATE3
    465. 

  465.  *	STATE4
    466. 

  466.  * output:
    467. 

  467.  *	STATE1:		finial state (output)
    468. 

  468.  *	STATE2
    469. 

  469.  *	STATE3
    470. 

  470.  *	STATE4
    471. 

  471.  * changed:
    472. 

  472.  *	KEY
    473. 

  473.  *	TKEYP (T1)
    474. 

  474.  */
    475. 

  475. _aesni_dec4:
    476. 

  476. 	movaps (KEYP), KEY		# key
    477. 

  477. 	mov KEYP, TKEYP
    478. 

  478. 	pxor KEY, STATE1		# round 0
    479. 

  479. 	pxor KEY, STATE2
    480. 

  480. 	pxor KEY, STATE3
    481. 

  481. 	pxor KEY, STATE4
    482. 

  482. 	add $0x30, TKEYP
    483. 

  483. 	cmp $24, KLEN
    484. 

  484. 	jb .L4dec128
    485. 

  485. 	lea 0x20(TKEYP), TKEYP
    486. 

  486. 	je .L4dec192
    487. 

  487. 	add $0x20, TKEYP
    488. 

  488. 	movaps -0x60(TKEYP), KEY
    489. 

  489. 	AESDEC KEY STATE1
    490. 

  490. 	AESDEC KEY STATE2
    491. 

  491. 	AESDEC KEY STATE3
    492. 

  492. 	AESDEC KEY STATE4
    493. 

  493. 	movaps -0x50(TKEYP), KEY
    494. 

  494. 	AESDEC KEY STATE1
    495. 

  495. 	AESDEC KEY STATE2
    496. 

  496. 	AESDEC KEY STATE3
    497. 

  497. 	AESDEC KEY STATE4
    498. 

  498. .align 4
    499. 

  499. .L4dec192:
    500. 

  500. 	movaps -0x40(TKEYP), KEY
    501. 

  501. 	AESDEC KEY STATE1
    502. 

  502. 	AESDEC KEY STATE2
    503. 

  503. 	AESDEC KEY STATE3
    504. 

  504. 	AESDEC KEY STATE4
    505. 

  505. 	movaps -0x30(TKEYP), KEY
    506. 

  506. 	AESDEC KEY STATE1
    507. 

  507. 	AESDEC KEY STATE2
    508. 

  508. 	AESDEC KEY STATE3
    509. 

  509. 	AESDEC KEY STATE4
    510. 

  510. .align 4
    511. 

  511. .L4dec128:
    512. 

  512. 	movaps -0x20(TKEYP), KEY
    513. 

  513. 	AESDEC KEY STATE1
    514. 

  514. 	AESDEC KEY STATE2
    515. 

  515. 	AESDEC KEY STATE3
    516. 

  516. 	AESDEC KEY STATE4
    517. 

  517. 	movaps -0x10(TKEYP), KEY
    518. 

  518. 	AESDEC KEY STATE1
    519. 

  519. 	AESDEC KEY STATE2
    520. 

  520. 	AESDEC KEY STATE3
    521. 

  521. 	AESDEC KEY STATE4
    522. 

  522. 	movaps (TKEYP), KEY
    523. 

  523. 	AESDEC KEY STATE1
    524. 

  524. 	AESDEC KEY STATE2
    525. 

  525. 	AESDEC KEY STATE3
    526. 

  526. 	AESDEC KEY STATE4
    527. 

  527. 	movaps 0x10(TKEYP), KEY
    528. 

  528. 	AESDEC KEY STATE1
    529. 

  529. 	AESDEC KEY STATE2
    530. 

  530. 	AESDEC KEY STATE3
    531. 

  531. 	AESDEC KEY STATE4
    532. 

  532. 	movaps 0x20(TKEYP), KEY
    533. 

  533. 	AESDEC KEY STATE1
    534. 

  534. 	AESDEC KEY STATE2
    535. 

  535. 	AESDEC KEY STATE3
    536. 

  536. 	AESDEC KEY STATE4
    537. 

  537. 	movaps 0x30(TKEYP), KEY
    538. 

  538. 	AESDEC KEY STATE1
    539. 

  539. 	AESDEC KEY STATE2
    540. 

  540. 	AESDEC KEY STATE3
    541. 

  541. 	AESDEC KEY STATE4
    542. 

  542. 	movaps 0x40(TKEYP), KEY
    543. 

  543. 	AESDEC KEY STATE1
    544. 

  544. 	AESDEC KEY STATE2
    545. 

  545. 	AESDEC KEY STATE3
    546. 

  546. 	AESDEC KEY STATE4
    547. 

  547. 	movaps 0x50(TKEYP), KEY
    548. 

  548. 	AESDEC KEY STATE1
    549. 

  549. 	AESDEC KEY STATE2
    550. 

  550. 	AESDEC KEY STATE3
    551. 

  551. 	AESDEC KEY STATE4
    552. 

  552. 	movaps 0x60(TKEYP), KEY
    553. 

  553. 	AESDEC KEY STATE1
    554. 

  554. 	AESDEC KEY STATE2
    555. 

  555. 	AESDEC KEY STATE3
    556. 

  556. 	AESDEC KEY STATE4
    557. 

  557. 	movaps 0x70(TKEYP), KEY
    558. 

  558. 	AESDECLAST KEY STATE1		# last round
    559. 

  559. 	AESDECLAST KEY STATE2
    560. 

  560. 	AESDECLAST KEY STATE3
    561. 

  561. 	AESDECLAST KEY STATE4
    562. 

  562. 	ret
    563. 

  563. 

  564. /*
    565. 

  565.  * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
    566. 

  566.  *		      size_t len)
    567. 

  567.  */
    568. 

  568. ENTRY(aesni_ecb_enc)
    569. 

  569. 	test LEN, LEN		# check length
    570. 

  570. 	jz .Lecb_enc_ret
    571. 

  571. 	mov 480(KEYP), KLEN
    572. 

  572. 	cmp $16, LEN
    573. 

  573. 	jb .Lecb_enc_ret
    574. 

  574. 	cmp $64, LEN
    575. 

  575. 	jb .Lecb_enc_loop1
    576. 

  576. .align 4
    577. 

  577. .Lecb_enc_loop4:
    578. 

  578. 	movups (INP), STATE1
    579. 

  579. 	movups 0x10(INP), STATE2
    580. 

  580. 	movups 0x20(INP), STATE3
    581. 

  581. 	movups 0x30(INP), STATE4
    582. 

  582. 	call _aesni_enc4
    583. 

  583. 	movups STATE1, (OUTP)
    584. 

  584. 	movups STATE2, 0x10(OUTP)
    585. 

  585. 	movups STATE3, 0x20(OUTP)
    586. 

  586. 	movups STATE4, 0x30(OUTP)
    587. 

  587. 	sub $64, LEN
    588. 

  588. 	add $64, INP
    589. 

  589. 	add $64, OUTP
    590. 

  590. 	cmp $64, LEN
    591. 

  591. 	jge .Lecb_enc_loop4
    592. 

  592. 	cmp $16, LEN
    593. 

  593. 	jb .Lecb_enc_ret
    594. 

  594. .align 4
    595. 

  595. .Lecb_enc_loop1:
    596. 

  596. 	movups (INP), STATE1
    597. 

  597. 	call _aesni_enc1
    598. 

  598. 	movups STATE1, (OUTP)
    599. 

  599. 	sub $16, LEN
    600. 

  600. 	add $16, INP
    601. 

  601. 	add $16, OUTP
    602. 

  602. 	cmp $16, LEN
    603. 

  603. 	jge .Lecb_enc_loop1
    604. 

  604. .Lecb_enc_ret:
    605. 

  605. 	ret
    606. 

  606. 

  607. /*
    608. 

  608.  * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
    609. 

  609.  *		      size_t len);
    610. 

  610.  */
    611. 

  611. ENTRY(aesni_ecb_dec)
    612. 

  612. 	test LEN, LEN
    613. 

  613. 	jz .Lecb_dec_ret
    614. 

  614. 	mov 480(KEYP), KLEN
    615. 

  615. 	add $240, KEYP
    616. 

  616. 	cmp $16, LEN
    617. 

  617. 	jb .Lecb_dec_ret
    618. 

  618. 	cmp $64, LEN
    619. 

  619. 	jb .Lecb_dec_loop1
    620. 

  620. .align 4
    621. 

  621. .Lecb_dec_loop4:
    622. 

  622. 	movups (INP), STATE1
    623. 

  623. 	movups 0x10(INP), STATE2
    624. 

  624. 	movups 0x20(INP), STATE3
    625. 

  625. 	movups 0x30(INP), STATE4
    626. 

  626. 	call _aesni_dec4
    627. 

  627. 	movups STATE1, (OUTP)
    628. 

  628. 	movups STATE2, 0x10(OUTP)
    629. 

  629. 	movups STATE3, 0x20(OUTP)
    630. 

  630. 	movups STATE4, 0x30(OUTP)
    631. 

  631. 	sub $64, LEN
    632. 

  632. 	add $64, INP
    633. 

  633. 	add $64, OUTP
    634. 

  634. 	cmp $64, LEN
    635. 

  635. 	jge .Lecb_dec_loop4
    636. 

  636. 	cmp $16, LEN
    637. 

  637. 	jb .Lecb_dec_ret
    638. 

  638. .align 4
    639. 

  639. .Lecb_dec_loop1:
    640. 

  640. 	movups (INP), STATE1
    641. 

  641. 	call _aesni_dec1
    642. 

  642. 	movups STATE1, (OUTP)
    643. 

  643. 	sub $16, LEN
    644. 

  644. 	add $16, INP
    645. 

  645. 	add $16, OUTP
    646. 

  646. 	cmp $16, LEN
    647. 

  647. 	jge .Lecb_dec_loop1
    648. 

  648. .Lecb_dec_ret:
    649. 

  649. 	ret
    650. 

  650. 

  651. /*
    652. 

  652.  * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
    653. 

  653.  *		      size_t len, u8 *iv)
    654. 

  654.  */
    655. 

  655. ENTRY(aesni_cbc_enc)
    656. 

  656. 	cmp $16, LEN
    657. 

  657. 	jb .Lcbc_enc_ret
    658. 

  658. 	mov 480(KEYP), KLEN
    659. 

  659. 	movups (IVP), STATE	# load iv as initial state
    660. 

  660. .align 4
    661. 

  661. .Lcbc_enc_loop:
    662. 

  662. 	movups (INP), IN	# load input
    663. 

  663. 	pxor IN, STATE
    664. 

  664. 	call _aesni_enc1
    665. 

  665. 	movups STATE, (OUTP)	# store output
    666. 

  666. 	sub $16, LEN
    667. 

  667. 	add $16, INP
    668. 

  668. 	add $16, OUTP
    669. 

  669. 	cmp $16, LEN
    670. 

  670. 	jge .Lcbc_enc_loop
    671. 

  671. 	movups STATE, (IVP)
    672. 

  672. .Lcbc_enc_ret:
    673. 

  673. 	ret
    674. 

  674. 

  675. /*
    676. 

  676.  * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
    677. 

  677.  *		      size_t len, u8 *iv)
    678. 

  678.  */
    679. 

  679. ENTRY(aesni_cbc_dec)
    680. 

  680. 	cmp $16, LEN
    681. 

  681. 	jb .Lcbc_dec_just_ret
    682. 

  682. 	mov 480(KEYP), KLEN
    683. 

  683. 	add $240, KEYP
    684. 

  684. 	movups (IVP), IV
    685. 

  685. 	cmp $64, LEN
    686. 

  686. 	jb .Lcbc_dec_loop1
    687. 

  687. .align 4
    688. 

  688. .Lcbc_dec_loop4:
    689. 

  689. 	movups (INP), IN1
    690. 

  690. 	movaps IN1, STATE1
    691. 

  691. 	movups 0x10(INP), IN2
    692. 

  692. 	movaps IN2, STATE2
    693. 

  693. 	movups 0x20(INP), IN3
    694. 

  694. 	movaps IN3, STATE3
    695. 

  695. 	movups 0x30(INP), IN4
    696. 

  696. 	movaps IN4, STATE4
    697. 

  697. 	call _aesni_dec4
    698. 

  698. 	pxor IV, STATE1
    699. 

  699. 	pxor IN1, STATE2
    700. 

  700. 	pxor IN2, STATE3
    701. 

  701. 	pxor IN3, STATE4
    702. 

  702. 	movaps IN4, IV
    703. 

  703. 	movups STATE1, (OUTP)
    704. 

  704. 	movups STATE2, 0x10(OUTP)
    705. 

  705. 	movups STATE3, 0x20(OUTP)
    706. 

  706. 	movups STATE4, 0x30(OUTP)
    707. 

  707. 	sub $64, LEN
    708. 

  708. 	add $64, INP
    709. 

  709. 	add $64, OUTP
    710. 

  710. 	cmp $64, LEN
    711. 

  711. 	jge .Lcbc_dec_loop4
    712. 

  712. 	cmp $16, LEN
    713. 

  713. 	jb .Lcbc_dec_ret
    714. 

  714. .align 4
    715. 

  715. .Lcbc_dec_loop1:
    716. 

  716. 	movups (INP), IN
    717. 

  717. 	movaps IN, STATE
    718. 

  718. 	call _aesni_dec1
    719. 

  719. 	pxor IV, STATE
    720. 

  720. 	movups STATE, (OUTP)
    721. 

  721. 	movaps IN, IV
    722. 

  722. 	sub $16, LEN
    723. 

  723. 	add $16, INP
    724. 

  724. 	add $16, OUTP
    725. 

  725. 	cmp $16, LEN
    726. 

  726. 	jge .Lcbc_dec_loop1
    727. 

  727. .Lcbc_dec_ret:
    728. 

  728. 	movups IV, (IVP)
    729. 

  729. .Lcbc_dec_just_ret:
    730. 

  730. 	ret
    731. 

  731. 

  732. .align 16
    733. 

  733. .Lbswap_mask:
    734. 

  734. 	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
    735. 

  735. 

  736. /*
    737. 

  737.  * _aesni_inc_init:	internal ABI
    738. 

  738.  *	setup registers used by _aesni_inc
    739. 

  739.  * input:
    740. 

  740.  *	IV
    741. 

  741.  * output:
    742. 

  742.  *	CTR:	== IV, in little endian
    743. 

  743.  *	TCTR_LOW: == lower qword of CTR
    744. 

  744.  *	INC:	== 1, in little endian
    745. 

  745.  *	BSWAP_MASK == endian swapping mask
    746. 

  746.  */
    747. 

  747. _aesni_inc_init:
    748. 

  748. 	movaps .Lbswap_mask, BSWAP_MASK
    749. 

  749. 	movaps IV, CTR
    750. 

  750. 	PSHUFB_XMM BSWAP_MASK CTR
    751. 

  751. 	mov $1, TCTR_LOW
    752. 

  752. 	MOVQ_R64_XMM TCTR_LOW INC
    753. 

  753. 	MOVQ_R64_XMM CTR TCTR_LOW
    754. 

  754. 	ret
    755. 

  755. 

  756. /*
    757. 

  757.  * _aesni_inc:		internal ABI
    758. 

  758.  *	Increase IV by 1, IV is in big endian
    759. 

  759.  * input:
    760. 

  760.  *	IV
    761. 

  761.  *	CTR:	== IV, in little endian
    762. 

  762.  *	TCTR_LOW: == lower qword of CTR
    763. 

  763.  *	INC:	== 1, in little endian
    764. 

  764.  *	BSWAP_MASK == endian swapping mask
    765. 

  765.  * output:
    766. 

  766.  *	IV:	Increase by 1
    767. 

  767.  * changed:
    768. 

  768.  *	CTR:	== output IV, in little endian
    769. 

  769.  *	TCTR_LOW: == lower qword of CTR
    770. 

  770.  */
    771. 

  771. _aesni_inc:
    772. 

  772. 	paddq INC, CTR
    773. 

  773. 	add $1, TCTR_LOW
    774. 

  774. 	jnc .Linc_low
    775. 

  775. 	pslldq $8, INC
    776. 

  776. 	paddq INC, CTR
    777. 

  777. 	psrldq $8, INC
    778. 

  778. .Linc_low:
    779. 

  779. 	movaps CTR, IV
    780. 

  780. 	PSHUFB_XMM BSWAP_MASK IV
    781. 

  781. 	ret
    782. 

  782. 

  783. /*
    784. 

  784.  * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
    785. 

  785.  *		      size_t len, u8 *iv)
    786. 

  786.  */
    787. 

  787. ENTRY(aesni_ctr_enc)
    788. 

  788. 	cmp $16, LEN
    789. 

  789. 	jb .Lctr_enc_just_ret
    790. 

  790. 	mov 480(KEYP), KLEN
    791. 

  791. 	movups (IVP), IV
    792. 

  792. 	call _aesni_inc_init
    793. 

  793. 	cmp $64, LEN
    794. 

  794. 	jb .Lctr_enc_loop1
    795. 

  795. .align 4
    796. 

  796. .Lctr_enc_loop4:
    797. 

  797. 	movaps IV, STATE1
    798. 

  798. 	call _aesni_inc
    799. 

  799. 	movups (INP), IN1
    800. 

  800. 	movaps IV, STATE2
    801. 

  801. 	call _aesni_inc
    802. 

  802. 	movups 0x10(INP), IN2
    803. 

  803. 	movaps IV, STATE3
    804. 

  804. 	call _aesni_inc
    805. 

  805. 	movups 0x20(INP), IN3
    806. 

  806. 	movaps IV, STATE4
    807. 

  807. 	call _aesni_inc
    808. 

  808. 	movups 0x30(INP), IN4
    809. 

  809. 	call _aesni_enc4
    810. 

  810. 	pxor IN1, STATE1
    811. 

  811. 	movups STATE1, (OUTP)
    812. 

  812. 	pxor IN2, STATE2
    813. 

  813. 	movups STATE2, 0x10(OUTP)
    814. 

  814. 	pxor IN3, STATE3
    815. 

  815. 	movups STATE3, 0x20(OUTP)
    816. 

  816. 	pxor IN4, STATE4
    817. 

  817. 	movups STATE4, 0x30(OUTP)
    818. 

  818. 	sub $64, LEN
    819. 

  819. 	add $64, INP
    820. 

  820. 	add $64, OUTP
    821. 

  821. 	cmp $64, LEN
    822. 

  822. 	jge .Lctr_enc_loop4
    823. 

  823. 	cmp $16, LEN
    824. 

  824. 	jb .Lctr_enc_ret
    825. 

  825. .align 4
    826. 

  826. .Lctr_enc_loop1:
    827. 

  827. 	movaps IV, STATE
    828. 

  828. 	call _aesni_inc
    829. 

  829. 	movups (INP), IN
    830. 

  830. 	call _aesni_enc1
    831. 

  831. 	pxor IN, STATE
    832. 

  832. 	movups STATE, (OUTP)
    833. 

  833. 	sub $16, LEN
    834. 

  834. 	add $16, INP
    835. 

  835. 	add $16, OUTP
    836. 

  836. 	cmp $16, LEN
    837. 

  837. 	jge .Lctr_enc_loop1
    838. 

  838. .Lctr_enc_ret:
    839. 

  839. 	movups IV, (IVP)
    840. 

  840. .Lctr_enc_just_ret:
    841. 

  841. 	ret
    842.