Grcov report - sha256_x86

1

// Copyright (c) 2018-2020 The Bitcoin Core developers

2

// Distributed under the MIT software license, see the accompanying

3

// file COPYING or http://www.opensource.org/licenses/mit-license.php.

4

//

5

// Based on https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-x86.c,

6

// Written and placed in public domain by Jeffrey Walton.

7

// Based on code from Intel, and by Sean Gulley for the miTLS project.

8

9

#ifdef ENABLE_X86_SHANI

10

11

#include <stdint.h>

12

#include <immintrin.h>

13

14

namespace {

15

16

alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c};

17

alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, 0x67, 0xe6, 0x09, 0x6a};

18

alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, 0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, 0x72, 0xf3, 0x6e, 0x3c};

19

20

void inline  __attribute__((always_inline)) QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0)

21

22

    const __m128i msg = _mm_set_epi64x(k1, k0);

23

    state1 = _mm_sha256rnds2_epu32(state1, state0, msg);

24

    state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e));

25

26

27

void inline  __attribute__((always_inline)) QuadRound(__m128i& state0, __m128i& state1, __m128i m, uint64_t k1, uint64_t k0)

28

29

    const __m128i msg = _mm_add_epi32(m, _mm_set_epi64x(k1, k0));

30

    state1 = _mm_sha256rnds2_epu32(state1, state0, msg);

31

    state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e));

32

33

34

void inline  __attribute__((always_inline)) ShiftMessageA(__m128i& m0, __m128i m1)

35

36

    m0 = _mm_sha256msg1_epu32(m0, m1);

37

38

39

void inline  __attribute__((always_inline)) ShiftMessageC(__m128i& m0, __m128i m1, __m128i& m2)

40

41

    m2 = _mm_sha256msg2_epu32(_mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)), m1);

42

43

44

void inline __attribute__((always_inline)) ShiftMessageB(__m128i& m0, __m128i m1, __m128i& m2)

45

46

    ShiftMessageC(m0, m1, m2);

47

    ShiftMessageA(m0, m1);

48

49

50

void inline __attribute__((always_inline)) Shuffle(__m128i& s0, __m128i& s1)

51

52

    const __m128i t1 = _mm_shuffle_epi32(s0, 0xB1);

53

    const __m128i t2 = _mm_shuffle_epi32(s1, 0x1B);

54

    s0 = _mm_alignr_epi8(t1, t2, 0x08);

55

    s1 = _mm_blend_epi16(t2, t1, 0xF0);

56

57

58

void inline __attribute__((always_inline)) Unshuffle(__m128i& s0, __m128i& s1)

59

60

    const __m128i t1 = _mm_shuffle_epi32(s0, 0x1B);

61

    const __m128i t2 = _mm_shuffle_epi32(s1, 0xB1);

62

    s0 = _mm_blend_epi16(t1, t2, 0xF0);

63

    s1 = _mm_alignr_epi8(t2, t1, 0x08);

64

65

66

__m128i inline  __attribute__((always_inline)) Load(const unsigned char* in)

67

68

    return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), _mm_load_si128((const __m128i*)MASK));

69

70

71

void inline  __attribute__((always_inline)) Save(unsigned char* out, __m128i s)

72

73

    _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, _mm_load_si128((const __m128i*)MASK)));

74

75

76

77

namespace sha256_x86_shani {

78

void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)

79

80

    __m128i m0, m1, m2, m3, s0, s1, so0, so1;

81

82

    /* Load state */

83

    s0 = _mm_loadu_si128((const __m128i*)s);

84

    s1 = _mm_loadu_si128((const __m128i*)(s + 4));

85

    Shuffle(s0, s1);

86

87

    while (blocks--) {

88

        /* Remember old state */

89

        so0 = s0;

90

        so1 = s1;

91

92

        /* Load data and transform */

93

        m0 = Load(chunk);

94

        QuadRound(s0, s1, m0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);

95

        m1 = Load(chunk + 16);

96

        QuadRound(s0, s1, m1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);

97

        ShiftMessageA(m0, m1);

98

        m2 = Load(chunk + 32);

99

        QuadRound(s0, s1, m2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);

100

        ShiftMessageA(m1, m2);

101

        m3 = Load(chunk + 48);

102

        QuadRound(s0, s1, m3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);

103

        ShiftMessageB(m2, m3, m0);

104

        QuadRound(s0, s1, m0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);

105

        ShiftMessageB(m3, m0, m1);

106

        QuadRound(s0, s1, m1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);

107

        ShiftMessageB(m0, m1, m2);

108

        QuadRound(s0, s1, m2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);

109

        ShiftMessageB(m1, m2, m3);

110

        QuadRound(s0, s1, m3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);

111

        ShiftMessageB(m2, m3, m0);

112

        QuadRound(s0, s1, m0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);

113

        ShiftMessageB(m3, m0, m1);

114

        QuadRound(s0, s1, m1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);

115

        ShiftMessageB(m0, m1, m2);

116

        QuadRound(s0, s1, m2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);

117

        ShiftMessageB(m1, m2, m3);

118

        QuadRound(s0, s1, m3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);

119

        ShiftMessageB(m2, m3, m0);

120

        QuadRound(s0, s1, m0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);

121

        ShiftMessageB(m3, m0, m1);

122

        QuadRound(s0, s1, m1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);

123

        ShiftMessageC(m0, m1, m2);

124

        QuadRound(s0, s1, m2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);

125

        ShiftMessageC(m1, m2, m3);

126

        QuadRound(s0, s1, m3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);

127

128

        /* Combine with old state */

129

        s0 = _mm_add_epi32(s0, so0);

130

        s1 = _mm_add_epi32(s1, so1);

131

132

        /* Advance */

133

        chunk += 64;

134

135

136

    Unshuffle(s0, s1);

137

    _mm_storeu_si128((__m128i*)s, s0);

138

    _mm_storeu_si128((__m128i*)(s + 4), s1);

139

140

141

142

namespace sha256d64_x86_shani {

143

144

void Transform_2way(unsigned char* out, const unsigned char* in)

145

146

    __m128i am0, am1, am2, am3, as0, as1, aso0, aso1;

147

    __m128i bm0, bm1, bm2, bm3, bs0, bs1, bso0, bso1;

148

149

    /* Transform 1 */

150

    bs0 = as0 = _mm_load_si128((const __m128i*)INIT0);

151

    bs1 = as1 = _mm_load_si128((const __m128i*)INIT1);

152

    am0 = Load(in);

153

    bm0 = Load(in + 64);

154

    QuadRound(as0, as1, am0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);

155

    QuadRound(bs0, bs1, bm0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);

156

    am1 = Load(in + 16);

157

    bm1 = Load(in + 80);

158

    QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);

159

    QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);

160

    ShiftMessageA(am0, am1);

161

    ShiftMessageA(bm0, bm1);

162

    am2 = Load(in + 32);

163

    bm2 = Load(in + 96);

164

    QuadRound(as0, as1, am2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);

165

    QuadRound(bs0, bs1, bm2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);

166

    ShiftMessageA(am1, am2);

167

    ShiftMessageA(bm1, bm2);

168

    am3 = Load(in + 48);

169

    bm3 = Load(in + 112);

170

    QuadRound(as0, as1, am3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);

171

    QuadRound(bs0, bs1, bm3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);

172

    ShiftMessageB(am2, am3, am0);

173

    ShiftMessageB(bm2, bm3, bm0);

174

    QuadRound(as0, as1, am0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);

175

    QuadRound(bs0, bs1, bm0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);

176

    ShiftMessageB(am3, am0, am1);

177

    ShiftMessageB(bm3, bm0, bm1);

178

    QuadRound(as0, as1, am1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);

179

    QuadRound(bs0, bs1, bm1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);

180

    ShiftMessageB(am0, am1, am2);

181

    ShiftMessageB(bm0, bm1, bm2);

182

    QuadRound(as0, as1, am2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);

183

    QuadRound(bs0, bs1, bm2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);

184

    ShiftMessageB(am1, am2, am3);

185

    ShiftMessageB(bm1, bm2, bm3);

186

    QuadRound(as0, as1, am3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);

187

    QuadRound(bs0, bs1, bm3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);

188

    ShiftMessageB(am2, am3, am0);

189

    ShiftMessageB(bm2, bm3, bm0);

190

    QuadRound(as0, as1, am0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);

191

    QuadRound(bs0, bs1, bm0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);

192

    ShiftMessageB(am3, am0, am1);

193

    ShiftMessageB(bm3, bm0, bm1);

194

    QuadRound(as0, as1, am1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);

195

    QuadRound(bs0, bs1, bm1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);

196

    ShiftMessageB(am0, am1, am2);

197

    ShiftMessageB(bm0, bm1, bm2);

198

    QuadRound(as0, as1, am2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);

199

    QuadRound(bs0, bs1, bm2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);

200

    ShiftMessageB(am1, am2, am3);

201

    ShiftMessageB(bm1, bm2, bm3);

202

    QuadRound(as0, as1, am3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);

203

    QuadRound(bs0, bs1, bm3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);

204

    ShiftMessageB(am2, am3, am0);

205

    ShiftMessageB(bm2, bm3, bm0);

206

    QuadRound(as0, as1, am0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);

207

    QuadRound(bs0, bs1, bm0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);

208

    ShiftMessageB(am3, am0, am1);

209

    ShiftMessageB(bm3, bm0, bm1);

210

    QuadRound(as0, as1, am1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);

211

    QuadRound(bs0, bs1, bm1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);

212

    ShiftMessageC(am0, am1, am2);

213

    ShiftMessageC(bm0, bm1, bm2);

214

    QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);

215

    QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);

216

    ShiftMessageC(am1, am2, am3);

217

    ShiftMessageC(bm1, bm2, bm3);

218

    QuadRound(as0, as1, am3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);

219

    QuadRound(bs0, bs1, bm3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);

220

    as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0));

221

    bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0));

222

    as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1));

223

    bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1));

224

225

    /* Transform 2 */

226

    aso0 = as0;

227

    bso0 = bs0;

228

    aso1 = as1;

229

    bso1 = bs1;

230

    QuadRound(as0, as1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull);

231

    QuadRound(bs0, bs1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull);

232

    QuadRound(as0, as1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);

233

    QuadRound(bs0, bs1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);

234

    QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);

235

    QuadRound(bs0, bs1, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);

236

    QuadRound(as0, as1, 0xc19bf3749bdc06a7ull, 0x80deb1fe72be5d74ull);

237

    QuadRound(bs0, bs1, 0xc19bf3749bdc06a7ull, 0x80deb1fe72be5d74ull);

238

    QuadRound(as0, as1, 0x240cf2540fe1edc6ull, 0xf0fe4786649b69c1ull);

239

    QuadRound(bs0, bs1, 0x240cf2540fe1edc6ull, 0xf0fe4786649b69c1ull);

240

    QuadRound(as0, as1, 0x16f988fa61b9411eull, 0x6cc984be4fe9346full);

241

    QuadRound(bs0, bs1, 0x16f988fa61b9411eull, 0x6cc984be4fe9346full);

242

    QuadRound(as0, as1, 0xb9d99ec7b019fc65ull, 0xa88e5a6df2c65152ull);

243

    QuadRound(bs0, bs1, 0xb9d99ec7b019fc65ull, 0xa88e5a6df2c65152ull);

244

    QuadRound(as0, as1, 0xc7353eb0fdb1232bull, 0xe70eeaa09a1231c3ull);

245

    QuadRound(bs0, bs1, 0xc7353eb0fdb1232bull, 0xe70eeaa09a1231c3ull);

246

    QuadRound(as0, as1, 0xdc1eeefd5a0f118full, 0xcb976d5f3069bad5ull);

247

    QuadRound(bs0, bs1, 0xdc1eeefd5a0f118full, 0xcb976d5f3069bad5ull);

248

    QuadRound(as0, as1, 0xe15d5b1658f4ca9dull, 0xde0b7a040a35b689ull);

249

    QuadRound(bs0, bs1, 0xe15d5b1658f4ca9dull, 0xde0b7a040a35b689ull);

250

    QuadRound(as0, as1, 0x6fab9537a507ea32ull, 0x37088980007f3e86ull);

251

    QuadRound(bs0, bs1, 0x6fab9537a507ea32ull, 0x37088980007f3e86ull);

252

    QuadRound(as0, as1, 0xc0bbbe37cdaa3b6dull, 0x0d8cd6f117406110ull);

253

    QuadRound(bs0, bs1, 0xc0bbbe37cdaa3b6dull, 0x0d8cd6f117406110ull);

254

    QuadRound(as0, as1, 0x6fd15ca70b02e931ull, 0xdb48a36383613bdaull);

255

    QuadRound(bs0, bs1, 0x6fd15ca70b02e931ull, 0xdb48a36383613bdaull);

256

    QuadRound(as0, as1, 0x6d4378906ed41a95ull, 0x31338431521afacaull);

257

    QuadRound(bs0, bs1, 0x6d4378906ed41a95ull, 0x31338431521afacaull);

258

    QuadRound(as0, as1, 0x532fb63cb5c9a0e6ull, 0x9eccabbdc39c91f2ull);

259

    QuadRound(bs0, bs1, 0x532fb63cb5c9a0e6ull, 0x9eccabbdc39c91f2ull);

260

    QuadRound(as0, as1, 0x4c191d76a4954b68ull, 0x07237ea3d2c741c6ull);

261

    QuadRound(bs0, bs1, 0x4c191d76a4954b68ull, 0x07237ea3d2c741c6ull);

262

    as0 = _mm_add_epi32(as0, aso0);

263

    bs0 = _mm_add_epi32(bs0, bso0);

264

    as1 = _mm_add_epi32(as1, aso1);

265

    bs1 = _mm_add_epi32(bs1, bso1);

266

267

    /* Extract hash */

268

    Unshuffle(as0, as1);

269

    Unshuffle(bs0, bs1);

270

    am0 = as0;

271

    bm0 = bs0;

272

    am1 = as1;

273

    bm1 = bs1;

274

275

    /* Transform 3 */

276

    bs0 = as0 = _mm_load_si128((const __m128i*)INIT0);

277

    bs1 = as1 = _mm_load_si128((const __m128i*)INIT1);

278

    QuadRound(as0, as1, am0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull);

279

    QuadRound(bs0, bs1, bm0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull);

280

    QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);

281

    QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);

282

    ShiftMessageA(am0, am1);

283

    ShiftMessageA(bm0, bm1);

284

    bm2 = am2 = _mm_set_epi64x(0x0ull, 0x80000000ull);

285

    QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b015807aa98ull);

286

    QuadRound(bs0, bs1, 0x550c7dc3243185beull, 0x12835b015807aa98ull);

287

    ShiftMessageA(am1, am2);

288

    ShiftMessageA(bm1, bm2);

289

    bm3 = am3 = _mm_set_epi64x(0x10000000000ull, 0x0ull);

290

    QuadRound(as0, as1, 0xc19bf2749bdc06a7ull, 0x80deb1fe72be5d74ull);

291

    QuadRound(bs0, bs1, 0xc19bf2749bdc06a7ull, 0x80deb1fe72be5d74ull);

292

    ShiftMessageB(am2, am3, am0);

293

    ShiftMessageB(bm2, bm3, bm0);

294

    QuadRound(as0, as1, am0, 0x240ca1cc0fc19dc6ull, 0xefbe4786e49b69c1ull);

295

    QuadRound(bs0, bs1, bm0, 0x240ca1cc0fc19dc6ull, 0xefbe4786e49b69c1ull);

296

    ShiftMessageB(am3, am0, am1);

297

    ShiftMessageB(bm3, bm0, bm1);

298

    QuadRound(as0, as1, am1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);

299

    QuadRound(bs0, bs1, bm1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);

300

    ShiftMessageB(am0, am1, am2);

301

    ShiftMessageB(bm0, bm1, bm2);

302

    QuadRound(as0, as1, am2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);

303

    QuadRound(bs0, bs1, bm2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);

304

    ShiftMessageB(am1, am2, am3);

305

    ShiftMessageB(bm1, bm2, bm3);

306

    QuadRound(as0, as1, am3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);

307

    QuadRound(bs0, bs1, bm3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);

308

    ShiftMessageB(am2, am3, am0);

309

    ShiftMessageB(bm2, bm3, bm0);

310

    QuadRound(as0, as1, am0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);

311

    QuadRound(bs0, bs1, bm0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);

312

    ShiftMessageB(am3, am0, am1);

313

    ShiftMessageB(bm3, bm0, bm1);

314

    QuadRound(as0, as1, am1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);

315

    QuadRound(bs0, bs1, bm1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);

316

    ShiftMessageB(am0, am1, am2);

317

    ShiftMessageB(bm0, bm1, bm2);

318

    QuadRound(as0, as1, am2, 0xc76c51a3c24b8b70ull, 0xa81a664ba2bfe8A1ull);

319

    QuadRound(bs0, bs1, bm2, 0xc76c51a3c24b8b70ull, 0xa81a664ba2bfe8A1ull);

320

    ShiftMessageB(am1, am2, am3);

321

    ShiftMessageB(bm1, bm2, bm3);

322

    QuadRound(as0, as1, am3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);

323

    QuadRound(bs0, bs1, bm3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);

324

    ShiftMessageB(am2, am3, am0);

325

    ShiftMessageB(bm2, bm3, bm0);

326

    QuadRound(as0, as1, am0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);

327

    QuadRound(bs0, bs1, bm0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);

328

    ShiftMessageB(am3, am0, am1);

329

    ShiftMessageB(bm3, bm0, bm1);

330

    QuadRound(as0, as1, am1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);

331

    QuadRound(bs0, bs1, bm1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);

332

    ShiftMessageC(am0, am1, am2);

333

    ShiftMessageC(bm0, bm1, bm2);

334

    QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);

335

    QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);

336

    ShiftMessageC(am1, am2, am3);

337

    ShiftMessageC(bm1, bm2, bm3);

338

    QuadRound(as0, as1, am3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull);

339

    QuadRound(bs0, bs1, bm3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull);

340

    as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0));

341

    bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0));

342

    as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1));

343

    bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1));

344

345

    /* Extract hash into out */

346

    Unshuffle(as0, as1);

347

    Unshuffle(bs0, bs1);

348

    Save(out, as0);

349

    Save(out + 16, as1);

350

    Save(out + 32, bs0);

351

    Save(out + 48, bs1);

352

353

354

355

356

#endif