Grcov report - sha256

1

// Copyright (c) 2017-2020 The Bitcoin Core developers

2

// Distributed under the MIT software license, see the accompanying

3

// file COPYING or http://www.opensource.org/licenses/mit-license.php.

4

//

5

// This is a translation to GCC extended asm syntax from YASM code by Intel

6

// (available at the bottom of this file).

7

8

#include <stdint.h>

9

#include <stdlib.h>

10

11

#if defined(__x86_64__) || defined(__amd64__)

12

13

namespace sha256_sse4

14

15

void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)

16

17

    static const uint32_t K256 alignas(16) [] = {

18

        0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,

19

        0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,

20

        0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,

21

        0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,

22

        0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,

23

        0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,

24

        0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,

25

        0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,

26

        0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,

27

        0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,

28

        0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,

29

        0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,

30

        0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,

31

        0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,

32

        0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,

33

        0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,

34

};

35

    static const uint32_t FLIP_MASK alignas(16) [] = {0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f};

36

    static const uint32_t SHUF_00BA alignas(16) [] = {0x03020100, 0x0b0a0908, 0xffffffff, 0xffffffff};

37

    static const uint32_t SHUF_DC00 alignas(16) [] = {0xffffffff, 0xffffffff, 0x03020100, 0x0b0a0908};

38

    uint32_t a, b, c, d, f, g, h, y0, y1, y2;

39

    uint64_t tbl;

40

    uint64_t inp_end, inp;

41

    uint32_t xfer alignas(16) [4];

42

43

    __asm__ __volatile__(

44

        "shl    $0x6,%2;"

45

        "je     Ldone_hash_%=;"

46

        "add    %1,%2;"

47

        "mov    %2,%14;"

48

        "mov    (%0),%3;"

49

        "mov    0x4(%0),%4;"

50

        "mov    0x8(%0),%5;"

51

        "mov    0xc(%0),%6;"

52

        "mov    0x10(%0),%k2;"

53

        "mov    0x14(%0),%7;"

54

        "mov    0x18(%0),%8;"

55

        "mov    0x1c(%0),%9;"

56

        "movdqa %18,%%xmm12;"

57

        "movdqa %19,%%xmm10;"

58

        "movdqa %20,%%xmm11;"

59

60

        "Lloop0_%=:"

61

        "lea    %17,%13;"

62

        "movdqu (%1),%%xmm4;"

63

        "pshufb %%xmm12,%%xmm4;"

64

        "movdqu 0x10(%1),%%xmm5;"

65

        "pshufb %%xmm12,%%xmm5;"

66

        "movdqu 0x20(%1),%%xmm6;"

67

        "pshufb %%xmm12,%%xmm6;"

68

        "movdqu 0x30(%1),%%xmm7;"

69

        "pshufb %%xmm12,%%xmm7;"

70

        "mov    %1,%15;"

71

        "mov    $3,%1;"

72

73

        "Lloop1_%=:"

74

        "movdqa 0x0(%13),%%xmm9;"

75

        "paddd  %%xmm4,%%xmm9;"

76

        "movdqa %%xmm9,%16;"

77

        "movdqa %%xmm7,%%xmm0;"

78

        "mov    %k2,%10;"

79

        "ror    $0xe,%10;"

80

        "mov    %3,%11;"

81

        "palignr $0x4,%%xmm6,%%xmm0;"

82

        "ror    $0x9,%11;"

83

        "xor    %k2,%10;"

84

        "mov    %7,%12;"

85

        "ror    $0x5,%10;"

86

        "movdqa %%xmm5,%%xmm1;"

87

        "xor    %3,%11;"

88

        "xor    %8,%12;"

89

        "paddd  %%xmm4,%%xmm0;"

90

        "xor    %k2,%10;"

91

        "and    %k2,%12;"

92

        "ror    $0xb,%11;"

93

        "palignr $0x4,%%xmm4,%%xmm1;"

94

        "xor    %3,%11;"

95

        "ror    $0x6,%10;"

96

        "xor    %8,%12;"

97

        "movdqa %%xmm1,%%xmm2;"

98

        "ror    $0x2,%11;"

99

        "add    %10,%12;"

100

        "add    %16,%12;"

101

        "movdqa %%xmm1,%%xmm3;"

102

        "mov    %3,%10;"

103

        "add    %12,%9;"

104

        "mov    %3,%12;"

105

        "pslld  $0x19,%%xmm1;"

106

        "or     %5,%10;"

107

        "add    %9,%6;"

108

        "and    %5,%12;"

109

        "psrld  $0x7,%%xmm2;"

110

        "and    %4,%10;"

111

        "add    %11,%9;"

112

        "por    %%xmm2,%%xmm1;"

113

        "or     %12,%10;"

114

        "add    %10,%9;"

115

        "movdqa %%xmm3,%%xmm2;"

116

        "mov    %6,%10;"

117

        "mov    %9,%11;"

118

        "movdqa %%xmm3,%%xmm8;"

119

        "ror    $0xe,%10;"

120

        "xor    %6,%10;"

121

        "mov    %k2,%12;"

122

        "ror    $0x9,%11;"

123

        "pslld  $0xe,%%xmm3;"

124

        "xor    %9,%11;"

125

        "ror    $0x5,%10;"

126

        "xor    %7,%12;"

127

        "psrld  $0x12,%%xmm2;"

128

        "ror    $0xb,%11;"

129

        "xor    %6,%10;"

130

        "and    %6,%12;"

131

        "ror    $0x6,%10;"

132

        "pxor   %%xmm3,%%xmm1;"

133

        "xor    %9,%11;"

134

        "xor    %7,%12;"

135

        "psrld  $0x3,%%xmm8;"

136

        "add    %10,%12;"

137

        "add    4+%16,%12;"

138

        "ror    $0x2,%11;"

139

        "pxor   %%xmm2,%%xmm1;"

140

        "mov    %9,%10;"

141

        "add    %12,%8;"

142

        "mov    %9,%12;"

143

        "pxor   %%xmm8,%%xmm1;"

144

        "or     %4,%10;"

145

        "add    %8,%5;"

146

        "and    %4,%12;"

147

        "pshufd $0xfa,%%xmm7,%%xmm2;"

148

        "and    %3,%10;"

149

        "add    %11,%8;"

150

        "paddd  %%xmm1,%%xmm0;"

151

        "or     %12,%10;"

152

        "add    %10,%8;"

153

        "movdqa %%xmm2,%%xmm3;"

154

        "mov    %5,%10;"

155

        "mov    %8,%11;"

156

        "ror    $0xe,%10;"

157

        "movdqa %%xmm2,%%xmm8;"

158

        "xor    %5,%10;"

159

        "ror    $0x9,%11;"

160

        "mov    %6,%12;"

161

        "xor    %8,%11;"

162

        "ror    $0x5,%10;"

163

        "psrlq  $0x11,%%xmm2;"

164

        "xor    %k2,%12;"

165

        "psrlq  $0x13,%%xmm3;"

166

        "xor    %5,%10;"

167

        "and    %5,%12;"

168

        "psrld  $0xa,%%xmm8;"

169

        "ror    $0xb,%11;"

170

        "xor    %8,%11;"

171

        "xor    %k2,%12;"

172

        "ror    $0x6,%10;"

173

        "pxor   %%xmm3,%%xmm2;"

174

        "add    %10,%12;"

175

        "ror    $0x2,%11;"

176

        "add    8+%16,%12;"

177

        "pxor   %%xmm2,%%xmm8;"

178

        "mov    %8,%10;"

179

        "add    %12,%7;"

180

        "mov    %8,%12;"

181

        "pshufb %%xmm10,%%xmm8;"

182

        "or     %3,%10;"

183

        "add    %7,%4;"

184

        "and    %3,%12;"

185

        "paddd  %%xmm8,%%xmm0;"

186

        "and    %9,%10;"

187

        "add    %11,%7;"

188

        "pshufd $0x50,%%xmm0,%%xmm2;"

189

        "or     %12,%10;"

190

        "add    %10,%7;"

191

        "movdqa %%xmm2,%%xmm3;"

192

        "mov    %4,%10;"

193

        "ror    $0xe,%10;"

194

        "mov    %7,%11;"

195

        "movdqa %%xmm2,%%xmm4;"

196

        "ror    $0x9,%11;"

197

        "xor    %4,%10;"

198

        "mov    %5,%12;"

199

        "ror    $0x5,%10;"

200

        "psrlq  $0x11,%%xmm2;"

201

        "xor    %7,%11;"

202

        "xor    %6,%12;"

203

        "psrlq  $0x13,%%xmm3;"

204

        "xor    %4,%10;"

205

        "and    %4,%12;"

206

        "ror    $0xb,%11;"

207

        "psrld  $0xa,%%xmm4;"

208

        "xor    %7,%11;"

209

        "ror    $0x6,%10;"

210

        "xor    %6,%12;"

211

        "pxor   %%xmm3,%%xmm2;"

212

        "ror    $0x2,%11;"

213

        "add    %10,%12;"

214

        "add    12+%16,%12;"

215

        "pxor   %%xmm2,%%xmm4;"

216

        "mov    %7,%10;"

217

        "add    %12,%k2;"

218

        "mov    %7,%12;"

219

        "pshufb %%xmm11,%%xmm4;"

220

        "or     %9,%10;"

221

        "add    %k2,%3;"

222

        "and    %9,%12;"

223

        "paddd  %%xmm0,%%xmm4;"

224

        "and    %8,%10;"

225

        "add    %11,%k2;"

226

        "or     %12,%10;"

227

        "add    %10,%k2;"

228

        "movdqa 0x10(%13),%%xmm9;"

229

        "paddd  %%xmm5,%%xmm9;"

230

        "movdqa %%xmm9,%16;"

231

        "movdqa %%xmm4,%%xmm0;"

232

        "mov    %3,%10;"

233

        "ror    $0xe,%10;"

234

        "mov    %k2,%11;"

235

        "palignr $0x4,%%xmm7,%%xmm0;"

236

        "ror    $0x9,%11;"

237

        "xor    %3,%10;"

238

        "mov    %4,%12;"

239

        "ror    $0x5,%10;"

240

        "movdqa %%xmm6,%%xmm1;"

241

        "xor    %k2,%11;"

242

        "xor    %5,%12;"

243

        "paddd  %%xmm5,%%xmm0;"

244

        "xor    %3,%10;"

245

        "and    %3,%12;"

246

        "ror    $0xb,%11;"

247

        "palignr $0x4,%%xmm5,%%xmm1;"

248

        "xor    %k2,%11;"

249

        "ror    $0x6,%10;"

250

        "xor    %5,%12;"

251

        "movdqa %%xmm1,%%xmm2;"

252

        "ror    $0x2,%11;"

253

        "add    %10,%12;"

254

        "add    %16,%12;"

255

        "movdqa %%xmm1,%%xmm3;"

256

        "mov    %k2,%10;"

257

        "add    %12,%6;"

258

        "mov    %k2,%12;"

259

        "pslld  $0x19,%%xmm1;"

260

        "or     %8,%10;"

261

        "add    %6,%9;"

262

        "and    %8,%12;"

263

        "psrld  $0x7,%%xmm2;"

264

        "and    %7,%10;"

265

        "add    %11,%6;"

266

        "por    %%xmm2,%%xmm1;"

267

        "or     %12,%10;"

268

        "add    %10,%6;"

269

        "movdqa %%xmm3,%%xmm2;"

270

        "mov    %9,%10;"

271

        "mov    %6,%11;"

272

        "movdqa %%xmm3,%%xmm8;"

273

        "ror    $0xe,%10;"

274

        "xor    %9,%10;"

275

        "mov    %3,%12;"

276

        "ror    $0x9,%11;"

277

        "pslld  $0xe,%%xmm3;"

278

        "xor    %6,%11;"

279

        "ror    $0x5,%10;"

280

        "xor    %4,%12;"

281

        "psrld  $0x12,%%xmm2;"

282

        "ror    $0xb,%11;"

283

        "xor    %9,%10;"

284

        "and    %9,%12;"

285

        "ror    $0x6,%10;"

286

        "pxor   %%xmm3,%%xmm1;"

287

        "xor    %6,%11;"

288

        "xor    %4,%12;"

289

        "psrld  $0x3,%%xmm8;"

290

        "add    %10,%12;"

291

        "add    4+%16,%12;"

292

        "ror    $0x2,%11;"

293

        "pxor   %%xmm2,%%xmm1;"

294

        "mov    %6,%10;"

295

        "add    %12,%5;"

296

        "mov    %6,%12;"

297

        "pxor   %%xmm8,%%xmm1;"

298

        "or     %7,%10;"

299

        "add    %5,%8;"

300

        "and    %7,%12;"

301

        "pshufd $0xfa,%%xmm4,%%xmm2;"

302

        "and    %k2,%10;"

303

        "add    %11,%5;"

304

        "paddd  %%xmm1,%%xmm0;"

305

        "or     %12,%10;"

306

        "add    %10,%5;"

307

        "movdqa %%xmm2,%%xmm3;"

308

        "mov    %8,%10;"

309

        "mov    %5,%11;"

310

        "ror    $0xe,%10;"

311

        "movdqa %%xmm2,%%xmm8;"

312

        "xor    %8,%10;"

313

        "ror    $0x9,%11;"

314

        "mov    %9,%12;"

315

        "xor    %5,%11;"

316

        "ror    $0x5,%10;"

317

        "psrlq  $0x11,%%xmm2;"

318

        "xor    %3,%12;"

319

        "psrlq  $0x13,%%xmm3;"

320

        "xor    %8,%10;"

321

        "and    %8,%12;"

322

        "psrld  $0xa,%%xmm8;"

323

        "ror    $0xb,%11;"

324

        "xor    %5,%11;"

325

        "xor    %3,%12;"

326

        "ror    $0x6,%10;"

327

        "pxor   %%xmm3,%%xmm2;"

328

        "add    %10,%12;"

329

        "ror    $0x2,%11;"

330

        "add    8+%16,%12;"

331

        "pxor   %%xmm2,%%xmm8;"

332

        "mov    %5,%10;"

333

        "add    %12,%4;"

334

        "mov    %5,%12;"

335

        "pshufb %%xmm10,%%xmm8;"

336

        "or     %k2,%10;"

337

        "add    %4,%7;"

338

        "and    %k2,%12;"

339

        "paddd  %%xmm8,%%xmm0;"

340

        "and    %6,%10;"

341

        "add    %11,%4;"

342

        "pshufd $0x50,%%xmm0,%%xmm2;"

343

        "or     %12,%10;"

344

        "add    %10,%4;"

345

        "movdqa %%xmm2,%%xmm3;"

346

        "mov    %7,%10;"

347

        "ror    $0xe,%10;"

348

        "mov    %4,%11;"

349

        "movdqa %%xmm2,%%xmm5;"

350

        "ror    $0x9,%11;"

351

        "xor    %7,%10;"

352

        "mov    %8,%12;"

353

        "ror    $0x5,%10;"

354

        "psrlq  $0x11,%%xmm2;"

355

        "xor    %4,%11;"

356

        "xor    %9,%12;"

357

        "psrlq  $0x13,%%xmm3;"

358

        "xor    %7,%10;"

359

        "and    %7,%12;"

360

        "ror    $0xb,%11;"

361

        "psrld  $0xa,%%xmm5;"

362

        "xor    %4,%11;"

363

        "ror    $0x6,%10;"

364

        "xor    %9,%12;"

365

        "pxor   %%xmm3,%%xmm2;"

366

        "ror    $0x2,%11;"

367

        "add    %10,%12;"

368

        "add    12+%16,%12;"

369

        "pxor   %%xmm2,%%xmm5;"

370

        "mov    %4,%10;"

371

        "add    %12,%3;"

372

        "mov    %4,%12;"

373

        "pshufb %%xmm11,%%xmm5;"

374

        "or     %6,%10;"

375

        "add    %3,%k2;"

376

        "and    %6,%12;"

377

        "paddd  %%xmm0,%%xmm5;"

378

        "and    %5,%10;"

379

        "add    %11,%3;"

380

        "or     %12,%10;"

381

        "add    %10,%3;"

382

        "movdqa 0x20(%13),%%xmm9;"

383

        "paddd  %%xmm6,%%xmm9;"

384

        "movdqa %%xmm9,%16;"

385

        "movdqa %%xmm5,%%xmm0;"

386

        "mov    %k2,%10;"

387

        "ror    $0xe,%10;"

388

        "mov    %3,%11;"

389

        "palignr $0x4,%%xmm4,%%xmm0;"

390

        "ror    $0x9,%11;"

391

        "xor    %k2,%10;"

392

        "mov    %7,%12;"

393

        "ror    $0x5,%10;"

394

        "movdqa %%xmm7,%%xmm1;"

395

        "xor    %3,%11;"

396

        "xor    %8,%12;"

397

        "paddd  %%xmm6,%%xmm0;"

398

        "xor    %k2,%10;"

399

        "and    %k2,%12;"

400

        "ror    $0xb,%11;"

401

        "palignr $0x4,%%xmm6,%%xmm1;"

402

        "xor    %3,%11;"

403

        "ror    $0x6,%10;"

404

        "xor    %8,%12;"

405

        "movdqa %%xmm1,%%xmm2;"

406

        "ror    $0x2,%11;"

407

        "add    %10,%12;"

408

        "add    %16,%12;"

409

        "movdqa %%xmm1,%%xmm3;"

410

        "mov    %3,%10;"

411

        "add    %12,%9;"

412

        "mov    %3,%12;"

413

        "pslld  $0x19,%%xmm1;"

414

        "or     %5,%10;"

415

        "add    %9,%6;"

416

        "and    %5,%12;"

417

        "psrld  $0x7,%%xmm2;"

418

        "and    %4,%10;"

419

        "add    %11,%9;"

420

        "por    %%xmm2,%%xmm1;"

421

        "or     %12,%10;"

422

        "add    %10,%9;"

423

        "movdqa %%xmm3,%%xmm2;"

424

        "mov    %6,%10;"

425

        "mov    %9,%11;"

426

        "movdqa %%xmm3,%%xmm8;"

427

        "ror    $0xe,%10;"

428

        "xor    %6,%10;"

429

        "mov    %k2,%12;"

430

        "ror    $0x9,%11;"

431

        "pslld  $0xe,%%xmm3;"

432

        "xor    %9,%11;"

433

        "ror    $0x5,%10;"

434

        "xor    %7,%12;"

435

        "psrld  $0x12,%%xmm2;"

436

        "ror    $0xb,%11;"

437

        "xor    %6,%10;"

438

        "and    %6,%12;"

439

        "ror    $0x6,%10;"

440

        "pxor   %%xmm3,%%xmm1;"

441

        "xor    %9,%11;"

442

        "xor    %7,%12;"

443

        "psrld  $0x3,%%xmm8;"

444

        "add    %10,%12;"

445

        "add    4+%16,%12;"

446

        "ror    $0x2,%11;"

447

        "pxor   %%xmm2,%%xmm1;"

448

        "mov    %9,%10;"

449

        "add    %12,%8;"

450

        "mov    %9,%12;"

451

        "pxor   %%xmm8,%%xmm1;"

452

        "or     %4,%10;"

453

        "add    %8,%5;"

454

        "and    %4,%12;"

455

        "pshufd $0xfa,%%xmm5,%%xmm2;"

456

        "and    %3,%10;"

457

        "add    %11,%8;"

458

        "paddd  %%xmm1,%%xmm0;"

459

        "or     %12,%10;"

460

        "add    %10,%8;"

461

        "movdqa %%xmm2,%%xmm3;"

462

        "mov    %5,%10;"

463

        "mov    %8,%11;"

464

        "ror    $0xe,%10;"

465

        "movdqa %%xmm2,%%xmm8;"

466

        "xor    %5,%10;"

467

        "ror    $0x9,%11;"

468

        "mov    %6,%12;"

469

        "xor    %8,%11;"

470

        "ror    $0x5,%10;"

471

        "psrlq  $0x11,%%xmm2;"

472

        "xor    %k2,%12;"

473

        "psrlq  $0x13,%%xmm3;"

474

        "xor    %5,%10;"

475

        "and    %5,%12;"

476

        "psrld  $0xa,%%xmm8;"

477

        "ror    $0xb,%11;"

478

        "xor    %8,%11;"

479

        "xor    %k2,%12;"

480

        "ror    $0x6,%10;"

481

        "pxor   %%xmm3,%%xmm2;"

482

        "add    %10,%12;"

483

        "ror    $0x2,%11;"

484

        "add    8+%16,%12;"

485

        "pxor   %%xmm2,%%xmm8;"

486

        "mov    %8,%10;"

487

        "add    %12,%7;"

488

        "mov    %8,%12;"

489

        "pshufb %%xmm10,%%xmm8;"

490

        "or     %3,%10;"

491

        "add    %7,%4;"

492

        "and    %3,%12;"

493

        "paddd  %%xmm8,%%xmm0;"

494

        "and    %9,%10;"

495

        "add    %11,%7;"

496

        "pshufd $0x50,%%xmm0,%%xmm2;"

497

        "or     %12,%10;"

498

        "add    %10,%7;"

499

        "movdqa %%xmm2,%%xmm3;"

500

        "mov    %4,%10;"

501

        "ror    $0xe,%10;"

502

        "mov    %7,%11;"

503

        "movdqa %%xmm2,%%xmm6;"

504

        "ror    $0x9,%11;"

505

        "xor    %4,%10;"

506

        "mov    %5,%12;"

507

        "ror    $0x5,%10;"

508

        "psrlq  $0x11,%%xmm2;"

509

        "xor    %7,%11;"

510

        "xor    %6,%12;"

511

        "psrlq  $0x13,%%xmm3;"

512

        "xor    %4,%10;"

513

        "and    %4,%12;"

514

        "ror    $0xb,%11;"

515

        "psrld  $0xa,%%xmm6;"

516

        "xor    %7,%11;"

517

        "ror    $0x6,%10;"

518

        "xor    %6,%12;"

519

        "pxor   %%xmm3,%%xmm2;"

520

        "ror    $0x2,%11;"

521

        "add    %10,%12;"

522

        "add    12+%16,%12;"

523

        "pxor   %%xmm2,%%xmm6;"

524

        "mov    %7,%10;"

525

        "add    %12,%k2;"

526

        "mov    %7,%12;"

527

        "pshufb %%xmm11,%%xmm6;"

528

        "or     %9,%10;"

529

        "add    %k2,%3;"

530

        "and    %9,%12;"

531

        "paddd  %%xmm0,%%xmm6;"

532

        "and    %8,%10;"

533

        "add    %11,%k2;"

534

        "or     %12,%10;"

535

        "add    %10,%k2;"

536

        "movdqa 0x30(%13),%%xmm9;"

537

        "paddd  %%xmm7,%%xmm9;"

538

        "movdqa %%xmm9,%16;"

539

        "add    $0x40,%13;"

540

        "movdqa %%xmm6,%%xmm0;"

541

        "mov    %3,%10;"

542

        "ror    $0xe,%10;"

543

        "mov    %k2,%11;"

544

        "palignr $0x4,%%xmm5,%%xmm0;"

545

        "ror    $0x9,%11;"

546

        "xor    %3,%10;"

547

        "mov    %4,%12;"

548

        "ror    $0x5,%10;"

549

        "movdqa %%xmm4,%%xmm1;"

550

        "xor    %k2,%11;"

551

        "xor    %5,%12;"

552

        "paddd  %%xmm7,%%xmm0;"

553

        "xor    %3,%10;"

554

        "and    %3,%12;"

555

        "ror    $0xb,%11;"

556

        "palignr $0x4,%%xmm7,%%xmm1;"

557

        "xor    %k2,%11;"

558

        "ror    $0x6,%10;"

559

        "xor    %5,%12;"

560

        "movdqa %%xmm1,%%xmm2;"

561

        "ror    $0x2,%11;"

562

        "add    %10,%12;"

563

        "add    %16,%12;"

564

        "movdqa %%xmm1,%%xmm3;"

565

        "mov    %k2,%10;"

566

        "add    %12,%6;"

567

        "mov    %k2,%12;"

568

        "pslld  $0x19,%%xmm1;"

569

        "or     %8,%10;"

570

        "add    %6,%9;"

571

        "and    %8,%12;"

572

        "psrld  $0x7,%%xmm2;"

573

        "and    %7,%10;"

574

        "add    %11,%6;"

575

        "por    %%xmm2,%%xmm1;"

576

        "or     %12,%10;"

577

        "add    %10,%6;"

578

        "movdqa %%xmm3,%%xmm2;"

579

        "mov    %9,%10;"

580

        "mov    %6,%11;"

581

        "movdqa %%xmm3,%%xmm8;"

582

        "ror    $0xe,%10;"

583

        "xor    %9,%10;"

584

        "mov    %3,%12;"

585

        "ror    $0x9,%11;"

586

        "pslld  $0xe,%%xmm3;"

587

        "xor    %6,%11;"

588

        "ror    $0x5,%10;"

589

        "xor    %4,%12;"

590

        "psrld  $0x12,%%xmm2;"

591

        "ror    $0xb,%11;"

592

        "xor    %9,%10;"

593

        "and    %9,%12;"

594

        "ror    $0x6,%10;"

595

        "pxor   %%xmm3,%%xmm1;"

596

        "xor    %6,%11;"

597

        "xor    %4,%12;"

598

        "psrld  $0x3,%%xmm8;"

599

        "add    %10,%12;"

600

        "add    4+%16,%12;"

601

        "ror    $0x2,%11;"

602

        "pxor   %%xmm2,%%xmm1;"

603

        "mov    %6,%10;"

604

        "add    %12,%5;"

605

        "mov    %6,%12;"

606

        "pxor   %%xmm8,%%xmm1;"

607

        "or     %7,%10;"

608

        "add    %5,%8;"

609

        "and    %7,%12;"

610

        "pshufd $0xfa,%%xmm6,%%xmm2;"

611

        "and    %k2,%10;"

612

        "add    %11,%5;"

613

        "paddd  %%xmm1,%%xmm0;"

614

        "or     %12,%10;"

615

        "add    %10,%5;"

616

        "movdqa %%xmm2,%%xmm3;"

617

        "mov    %8,%10;"

618

        "mov    %5,%11;"

619

        "ror    $0xe,%10;"

620

        "movdqa %%xmm2,%%xmm8;"

621

        "xor    %8,%10;"

622

        "ror    $0x9,%11;"

623

        "mov    %9,%12;"

624

        "xor    %5,%11;"

625

        "ror    $0x5,%10;"

626

        "psrlq  $0x11,%%xmm2;"

627

        "xor    %3,%12;"

628

        "psrlq  $0x13,%%xmm3;"

629

        "xor    %8,%10;"

630

        "and    %8,%12;"

631

        "psrld  $0xa,%%xmm8;"

632

        "ror    $0xb,%11;"

633

        "xor    %5,%11;"

634

        "xor    %3,%12;"

635

        "ror    $0x6,%10;"

636

        "pxor   %%xmm3,%%xmm2;"

637

        "add    %10,%12;"

638

        "ror    $0x2,%11;"

639

        "add    8+%16,%12;"

640

        "pxor   %%xmm2,%%xmm8;"

641

        "mov    %5,%10;"

642

        "add    %12,%4;"

643

        "mov    %5,%12;"

644

        "pshufb %%xmm10,%%xmm8;"

645

        "or     %k2,%10;"

646

        "add    %4,%7;"

647

        "and    %k2,%12;"

648

        "paddd  %%xmm8,%%xmm0;"

649

        "and    %6,%10;"

650

        "add    %11,%4;"

651

        "pshufd $0x50,%%xmm0,%%xmm2;"

652

        "or     %12,%10;"

653

        "add    %10,%4;"

654

        "movdqa %%xmm2,%%xmm3;"

655

        "mov    %7,%10;"

656

        "ror    $0xe,%10;"

657

        "mov    %4,%11;"

658

        "movdqa %%xmm2,%%xmm7;"

659

        "ror    $0x9,%11;"

660

        "xor    %7,%10;"

661

        "mov    %8,%12;"

662

        "ror    $0x5,%10;"

663

        "psrlq  $0x11,%%xmm2;"

664

        "xor    %4,%11;"

665

        "xor    %9,%12;"

666

        "psrlq  $0x13,%%xmm3;"

667

        "xor    %7,%10;"

668

        "and    %7,%12;"

669

        "ror    $0xb,%11;"

670

        "psrld  $0xa,%%xmm7;"

671

        "xor    %4,%11;"

672

        "ror    $0x6,%10;"

673

        "xor    %9,%12;"

674

        "pxor   %%xmm3,%%xmm2;"

675

        "ror    $0x2,%11;"

676

        "add    %10,%12;"

677

        "add    12+%16,%12;"

678

        "pxor   %%xmm2,%%xmm7;"

679

        "mov    %4,%10;"

680

        "add    %12,%3;"

681

        "mov    %4,%12;"

682

        "pshufb %%xmm11,%%xmm7;"

683

        "or     %6,%10;"

684

        "add    %3,%k2;"

685

        "and    %6,%12;"

686

        "paddd  %%xmm0,%%xmm7;"

687

        "and    %5,%10;"

688

        "add    %11,%3;"

689

        "or     %12,%10;"

690

        "add    %10,%3;"

691

        "sub    $0x1,%1;"

692

        "jne    Lloop1_%=;"

693

        "mov    $0x2,%1;"

694

695

        "Lloop2_%=:"

696

        "paddd  0x0(%13),%%xmm4;"

697

        "movdqa %%xmm4,%16;"

698

        "mov    %k2,%10;"

699

        "ror    $0xe,%10;"

700

        "mov    %3,%11;"

701

        "xor    %k2,%10;"

702

        "ror    $0x9,%11;"

703

        "mov    %7,%12;"

704

        "xor    %3,%11;"

705

        "ror    $0x5,%10;"

706

        "xor    %8,%12;"

707

        "xor    %k2,%10;"

708

        "ror    $0xb,%11;"

709

        "and    %k2,%12;"

710

        "xor    %3,%11;"

711

        "ror    $0x6,%10;"

712

        "xor    %8,%12;"

713

        "add    %10,%12;"

714

        "ror    $0x2,%11;"

715

        "add    %16,%12;"

716

        "mov    %3,%10;"

717

        "add    %12,%9;"

718

        "mov    %3,%12;"

719

        "or     %5,%10;"

720

        "add    %9,%6;"

721

        "and    %5,%12;"

722

        "and    %4,%10;"

723

        "add    %11,%9;"

724

        "or     %12,%10;"

725

        "add    %10,%9;"

726

        "mov    %6,%10;"

727

        "ror    $0xe,%10;"

728

        "mov    %9,%11;"

729

        "xor    %6,%10;"

730

        "ror    $0x9,%11;"

731

        "mov    %k2,%12;"

732

        "xor    %9,%11;"

733

        "ror    $0x5,%10;"

734

        "xor    %7,%12;"

735

        "xor    %6,%10;"

736

        "ror    $0xb,%11;"

737

        "and    %6,%12;"

738

        "xor    %9,%11;"

739

        "ror    $0x6,%10;"

740

        "xor    %7,%12;"

741

        "add    %10,%12;"

742

        "ror    $0x2,%11;"

743

        "add    4+%16,%12;"

744

        "mov    %9,%10;"

745

        "add    %12,%8;"

746

        "mov    %9,%12;"

747

        "or     %4,%10;"

748

        "add    %8,%5;"

749

        "and    %4,%12;"

750

        "and    %3,%10;"

751

        "add    %11,%8;"

752

        "or     %12,%10;"

753

        "add    %10,%8;"

754

        "mov    %5,%10;"

755

        "ror    $0xe,%10;"

756

        "mov    %8,%11;"

757

        "xor    %5,%10;"

758

        "ror    $0x9,%11;"

759

        "mov    %6,%12;"

760

        "xor    %8,%11;"

761

        "ror    $0x5,%10;"

762

        "xor    %k2,%12;"

763

        "xor    %5,%10;"

764

        "ror    $0xb,%11;"

765

        "and    %5,%12;"

766

        "xor    %8,%11;"

767

        "ror    $0x6,%10;"

768

        "xor    %k2,%12;"

769

        "add    %10,%12;"

770

        "ror    $0x2,%11;"

771

        "add    8+%16,%12;"

772

        "mov    %8,%10;"

773

        "add    %12,%7;"

774

        "mov    %8,%12;"

775

        "or     %3,%10;"

776

        "add    %7,%4;"

777

        "and    %3,%12;"

778

        "and    %9,%10;"

779

        "add    %11,%7;"

780

        "or     %12,%10;"

781

        "add    %10,%7;"

782

        "mov    %4,%10;"

783

        "ror    $0xe,%10;"

784

        "mov    %7,%11;"

785

        "xor    %4,%10;"

786

        "ror    $0x9,%11;"

787

        "mov    %5,%12;"

788

        "xor    %7,%11;"

789

        "ror    $0x5,%10;"

790

        "xor    %6,%12;"

791

        "xor    %4,%10;"

792

        "ror    $0xb,%11;"

793

        "and    %4,%12;"

794

        "xor    %7,%11;"

795

        "ror    $0x6,%10;"

796

        "xor    %6,%12;"

797

        "add    %10,%12;"

798

        "ror    $0x2,%11;"

799

        "add    12+%16,%12;"

800

        "mov    %7,%10;"

801

        "add    %12,%k2;"

802

        "mov    %7,%12;"

803

        "or     %9,%10;"

804

        "add    %k2,%3;"

805

        "and    %9,%12;"

806

        "and    %8,%10;"

807

        "add    %11,%k2;"

808

        "or     %12,%10;"

809

        "add    %10,%k2;"

810

        "paddd  0x10(%13),%%xmm5;"

811

        "movdqa %%xmm5,%16;"

812

        "add    $0x20,%13;"

813

        "mov    %3,%10;"

814

        "ror    $0xe,%10;"

815

        "mov    %k2,%11;"

816

        "xor    %3,%10;"

817

        "ror    $0x9,%11;"

818

        "mov    %4,%12;"

819

        "xor    %k2,%11;"

820

        "ror    $0x5,%10;"

821

        "xor    %5,%12;"

822

        "xor    %3,%10;"

823

        "ror    $0xb,%11;"

824

        "and    %3,%12;"

825

        "xor    %k2,%11;"

826

        "ror    $0x6,%10;"

827

        "xor    %5,%12;"

828

        "add    %10,%12;"

829

        "ror    $0x2,%11;"

830

        "add    %16,%12;"

831

        "mov    %k2,%10;"

832

        "add    %12,%6;"

833

        "mov    %k2,%12;"

834

        "or     %8,%10;"

835

        "add    %6,%9;"

836

        "and    %8,%12;"

837

        "and    %7,%10;"

838

        "add    %11,%6;"

839

        "or     %12,%10;"

840

        "add    %10,%6;"

841

        "mov    %9,%10;"

842

        "ror    $0xe,%10;"

843

        "mov    %6,%11;"

844

        "xor    %9,%10;"

845

        "ror    $0x9,%11;"

846

        "mov    %3,%12;"

847

        "xor    %6,%11;"

848

        "ror    $0x5,%10;"

849

        "xor    %4,%12;"

850

        "xor    %9,%10;"

851

        "ror    $0xb,%11;"

852

        "and    %9,%12;"

853

        "xor    %6,%11;"

854

        "ror    $0x6,%10;"

855

        "xor    %4,%12;"

856

        "add    %10,%12;"

857

        "ror    $0x2,%11;"

858

        "add    4+%16,%12;"

859

        "mov    %6,%10;"

860

        "add    %12,%5;"

861

        "mov    %6,%12;"

862

        "or     %7,%10;"

863

        "add    %5,%8;"

864

        "and    %7,%12;"

865

        "and    %k2,%10;"

866

        "add    %11,%5;"

867

        "or     %12,%10;"

868

        "add    %10,%5;"

869

        "mov    %8,%10;"

870

        "ror    $0xe,%10;"

871

        "mov    %5,%11;"

872

        "xor    %8,%10;"

873

        "ror    $0x9,%11;"

874

        "mov    %9,%12;"

875

        "xor    %5,%11;"

876

        "ror    $0x5,%10;"

877

        "xor    %3,%12;"

878

        "xor    %8,%10;"

879

        "ror    $0xb,%11;"

880

        "and    %8,%12;"

881

        "xor    %5,%11;"

882

        "ror    $0x6,%10;"

883

        "xor    %3,%12;"

884

        "add    %10,%12;"

885

        "ror    $0x2,%11;"

886

        "add    8+%16,%12;"

887

        "mov    %5,%10;"

888

        "add    %12,%4;"

889

        "mov    %5,%12;"

890

        "or     %k2,%10;"

891

        "add    %4,%7;"

892

        "and    %k2,%12;"

893

        "and    %6,%10;"

894

        "add    %11,%4;"

895

        "or     %12,%10;"

896

        "add    %10,%4;"

897

        "mov    %7,%10;"

898

        "ror    $0xe,%10;"

899

        "mov    %4,%11;"

900

        "xor    %7,%10;"

901

        "ror    $0x9,%11;"

902

        "mov    %8,%12;"

903

        "xor    %4,%11;"

904

        "ror    $0x5,%10;"

905

        "xor    %9,%12;"

906

        "xor    %7,%10;"

907

        "ror    $0xb,%11;"

908

        "and    %7,%12;"

909

        "xor    %4,%11;"

910

        "ror    $0x6,%10;"

911

        "xor    %9,%12;"

912

        "add    %10,%12;"

913

        "ror    $0x2,%11;"

914

        "add    12+%16,%12;"

915

        "mov    %4,%10;"

916

        "add    %12,%3;"

917

        "mov    %4,%12;"

918

        "or     %6,%10;"

919

        "add    %3,%k2;"

920

        "and    %6,%12;"

921

        "and    %5,%10;"

922

        "add    %11,%3;"

923

        "or     %12,%10;"

924

        "add    %10,%3;"

925

        "movdqa %%xmm6,%%xmm4;"

926

        "movdqa %%xmm7,%%xmm5;"

927

        "sub    $0x1,%1;"

928

        "jne    Lloop2_%=;"

929

        "add    (%0),%3;"

930

        "mov    %3,(%0);"

931

        "add    0x4(%0),%4;"

932

        "mov    %4,0x4(%0);"

933

        "add    0x8(%0),%5;"

934

        "mov    %5,0x8(%0);"

935

        "add    0xc(%0),%6;"

936

        "mov    %6,0xc(%0);"

937

        "add    0x10(%0),%k2;"

938

        "mov    %k2,0x10(%0);"

939

        "add    0x14(%0),%7;"

940

        "mov    %7,0x14(%0);"

941

        "add    0x18(%0),%8;"

942

        "mov    %8,0x18(%0);"

943

        "add    0x1c(%0),%9;"

944

        "mov    %9,0x1c(%0);"

945

        "mov    %15,%1;"

946

        "add    $0x40,%1;"

947

        "cmp    %14,%1;"

948

        "jne    Lloop0_%=;"

949

950

        "Ldone_hash_%=:"

951

952

        : "+r"(s), "+r"(chunk), "+r"(blocks), "=r"(a), "=r"(b), "=r"(c), "=r"(d), /* e = chunk */ "=r"(f), "=r"(g), "=r"(h), "=r"(y0), "=r"(y1), "=r"(y2), "=r"(tbl), "+m"(inp_end), "+m"(inp), "+m"(xfer)

953

        : "m"(K256), "m"(FLIP_MASK), "m"(SHUF_00BA), "m"(SHUF_DC00)

954

        : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12"

955

);

956

957

958

959

/*

960

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

961

; Copyright (c) 2012, Intel Corporation

962

963

; All rights reserved.

964

965

; Redistribution and use in source and binary forms, with or without

966

; modification, are permitted provided that the following conditions are

967

; met:

968

969

; * Redistributions of source code must retain the above copyright

970

;   notice, this list of conditions and the following disclaimer.

971

972

; * Redistributions in binary form must reproduce the above copyright

973

;   notice, this list of conditions and the following disclaimer in the

974

;   documentation and/or other materials provided with the

975

;   distribution.

976

977

; * Neither the name of the Intel Corporation nor the names of its

978

;   contributors may be used to endorse or promote products derived from

979

;   this software without specific prior written permission.

980

981

982

; THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS" AND ANY

983

; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

984

; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

985

; PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR

986

; CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

987

; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

988

; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

989

; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF

990

; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING

991

; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS

992

; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

993

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

994

995

; Example YASM command lines:

996

; Windows:  yasm -Xvc -f x64 -rnasm -pnasm -o sha256_sse4.obj -g cv8 sha256_sse4.asm

997

; Linux:    yasm -f x64 -f elf64 -X gnu -g dwarf2 -D LINUX -o sha256_sse4.o sha256_sse4.asm

998

999

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

1000

1001

; This code is described in an Intel White-Paper:

1002

; "Fast SHA-256 Implementations on Intel Architecture Processors"

1003

1004

; To find it, surf to https://www.intel.com/p/en_US/embedded

1005

; and search for that title.

1006

; The paper is expected to be released roughly at the end of April, 2012

1007

1008

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

1009

; This code schedules 1 blocks at a time, with 4 lanes per block

1010

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

1011

1012

%define	MOVDQ movdqu ;; assume buffers not aligned

1013

1014

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Define Macros

1015

1016

; addm [mem], reg

1017

; Add reg to mem using reg-mem add and store

1018

%macro addm 2

1019

    add	%2, %1

1020

    mov	%1, %2

1021

%endm

1022

1023

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

1024

1025

; COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask

1026

; Load xmm with mem and byte swap each dword

1027

%macro COPY_XMM_AND_BSWAP 3

1028

    MOVDQ %1, %2

1029

    pshufb %1, %3

1030

%endmacro

1031

1032

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

1033

1034

%define X0 xmm4

1035

%define X1 xmm5

1036

%define X2 xmm6

1037

%define X3 xmm7

1038

1039

%define XTMP0 xmm0

1040

%define XTMP1 xmm1

1041

%define XTMP2 xmm2

1042

%define XTMP3 xmm3

1043

%define XTMP4 xmm8

1044

%define XFER  xmm9

1045

1046

%define SHUF_00BA	xmm10 ; shuffle xBxA -> 00BA

1047

%define SHUF_DC00	xmm11 ; shuffle xDxC -> DC00

1048

%define BYTE_FLIP_MASK	xmm12

1049

1050

%ifdef LINUX

1051

%define NUM_BLKS rdx	; 3rd arg

1052

%define CTX	rsi	; 2nd arg

1053

%define INP	rdi	; 1st arg

1054

1055

%define SRND	rdi	; clobbers INP

1056

%define c	ecx

1057

%define d 	r8d

1058

%define e 	edx

1059

%else

1060

%define NUM_BLKS r8	; 3rd arg

1061

%define CTX	rdx 	; 2nd arg

1062

%define INP	rcx 	; 1st arg

1063

1064

%define SRND	rcx	; clobbers INP

1065

%define c 	edi

1066

%define d	esi

1067

%define e 	r8d

1068

1069

%endif

1070

%define TBL	rbp

1071

%define a eax

1072

%define b ebx

1073

1074

%define f r9d

1075

%define g r10d

1076

%define h r11d

1077

1078

%define y0 r13d

1079

%define y1 r14d

1080

%define y2 r15d

1081

1082

1083

1084

_INP_END_SIZE	equ 8

1085

_INP_SIZE	equ 8

1086

_XFER_SIZE	equ 8

1087

%ifdef LINUX

1088

_XMM_SAVE_SIZE	equ 0

1089

%else

1090

_XMM_SAVE_SIZE	equ 7*16

1091

%endif

1092

; STACK_SIZE plus pushes must be an odd multiple of 8

1093

_ALIGN_SIZE	equ 8

1094

1095

_INP_END	equ 0

1096

_INP		equ _INP_END  + _INP_END_SIZE

1097

_XFER		equ _INP      + _INP_SIZE

1098

_XMM_SAVE	equ _XFER     + _XFER_SIZE + _ALIGN_SIZE

1099

STACK_SIZE	equ _XMM_SAVE + _XMM_SAVE_SIZE

1100

1101

; rotate_Xs

1102

; Rotate values of symbols X0...X3

1103

%macro rotate_Xs 0

1104

%xdefine X_ X0

1105

%xdefine X0 X1

1106

%xdefine X1 X2

1107

%xdefine X2 X3

1108

%xdefine X3 X_

1109

%endm

1110

1111

; ROTATE_ARGS

1112

; Rotate values of symbols a...h

1113

%macro ROTATE_ARGS 0

1114

%xdefine TMP_ h

1115

%xdefine h g

1116

%xdefine g f

1117

%xdefine f e

1118

%xdefine e d

1119

%xdefine d c

1120

%xdefine c b

1121

%xdefine b a

1122

%xdefine a TMP_

1123

%endm

1124

1125

%macro FOUR_ROUNDS_AND_SCHED 0

1126

	;; compute s0 four at a time and s1 two at a time

1127

	;; compute W[-16] + W[-7] 4 at a time

1128

	movdqa	XTMP0, X3

1129

    mov	y0, e		; y0 = e

1130

    ror	y0, (25-11)	; y0 = e >> (25-11)

1131

    mov	y1, a		; y1 = a

1132

	palignr	XTMP0, X2, 4	; XTMP0 = W[-7]

1133

    ror	y1, (22-13)	; y1 = a >> (22-13)

1134

    xor	y0, e		; y0 = e ^ (e >> (25-11))

1135

    mov	y2, f		; y2 = f

1136

    ror	y0, (11-6)	; y0 = (e >> (11-6)) ^ (e >> (25-6))

1137

	movdqa	XTMP1, X1

1138

    xor	y1, a		; y1 = a ^ (a >> (22-13)

1139

    xor	y2, g		; y2 = f^g

1140

	paddd	XTMP0, X0	; XTMP0 = W[-7] + W[-16]

1141

    xor	y0, e		; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))

1142

    and	y2, e		; y2 = (f^g)&e

1143

    ror	y1, (13-2)	; y1 = (a >> (13-2)) ^ (a >> (22-2))

1144

	;; compute s0

1145

	palignr	XTMP1, X0, 4	; XTMP1 = W[-15]

1146

    xor	y1, a		; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))

1147

    ror	y0, 6		; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)

1148

    xor	y2, g		; y2 = CH = ((f^g)&e)^g

1149

	movdqa	XTMP2, XTMP1	; XTMP2 = W[-15]

1150

    ror	y1, 2		; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)

1151

    add	y2, y0		; y2 = S1 + CH

1152

    add	y2, [rsp + _XFER + 0*4]	; y2 = k + w + S1 + CH

1153

	movdqa	XTMP3, XTMP1	; XTMP3 = W[-15]

1154

    mov	y0, a		; y0 = a

1155

    add	h, y2		; h = h + S1 + CH + k + w

1156

    mov	y2, a		; y2 = a

1157

	pslld	XTMP1, (32-7)

1158

    or	y0, c		; y0 = a|c

1159

    add	d, h		; d = d + h + S1 + CH + k + w

1160

    and	y2, c		; y2 = a&c

1161

	psrld	XTMP2, 7

1162

    and	y0, b		; y0 = (a|c)&b

1163

    add	h, y1		; h = h + S1 + CH + k + w + S0

1164

	por	XTMP1, XTMP2	; XTMP1 = W[-15] ror 7

1165

    or	y0, y2		; y0 = MAJ = (a|c)&b)|(a&c)

1166

    add	h, y0		; h = h + S1 + CH + k + w + S0 + MAJ

1167

1168

ROTATE_ARGS

1169

	movdqa	XTMP2, XTMP3	; XTMP2 = W[-15]

1170

    mov	y0, e		; y0 = e

1171

    mov	y1, a		; y1 = a

1172

	movdqa	XTMP4, XTMP3	; XTMP4 = W[-15]

1173

    ror	y0, (25-11)	; y0 = e >> (25-11)

1174

    xor	y0, e		; y0 = e ^ (e >> (25-11))

1175

    mov	y2, f		; y2 = f

1176

    ror	y1, (22-13)	; y1 = a >> (22-13)

1177

	pslld	XTMP3, (32-18)

1178

    xor	y1, a		; y1 = a ^ (a >> (22-13)

1179

    ror	y0, (11-6)	; y0 = (e >> (11-6)) ^ (e >> (25-6))

1180

    xor	y2, g		; y2 = f^g

1181

	psrld	XTMP2, 18

1182

    ror	y1, (13-2)	; y1 = (a >> (13-2)) ^ (a >> (22-2))

1183

    xor	y0, e		; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))

1184

    and	y2, e		; y2 = (f^g)&e

1185

    ror	y0, 6		; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)

1186

	pxor	XTMP1, XTMP3

1187

    xor	y1, a		; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))

1188

    xor	y2, g		; y2 = CH = ((f^g)&e)^g

1189

	psrld	XTMP4, 3	; XTMP4 = W[-15] >> 3

1190

    add	y2, y0		; y2 = S1 + CH

1191

    add	y2, [rsp + _XFER + 1*4]	; y2 = k + w + S1 + CH

1192

    ror	y1, 2		; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)

1193

	pxor	XTMP1, XTMP2	; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18

1194

    mov	y0, a		; y0 = a

1195

    add	h, y2		; h = h + S1 + CH + k + w

1196

    mov	y2, a		; y2 = a

1197

	pxor	XTMP1, XTMP4	; XTMP1 = s0

1198

    or	y0, c		; y0 = a|c

1199

    add	d, h		; d = d + h + S1 + CH + k + w

1200

    and	y2, c		; y2 = a&c

1201

	;; compute low s1

1202

	pshufd	XTMP2, X3, 11111010b	; XTMP2 = W[-2] {BBAA}

1203

    and	y0, b		; y0 = (a|c)&b

1204

    add	h, y1		; h = h + S1 + CH + k + w + S0

1205

	paddd	XTMP0, XTMP1	; XTMP0 = W[-16] + W[-7] + s0

1206

    or	y0, y2		; y0 = MAJ = (a|c)&b)|(a&c)

1207

    add	h, y0		; h = h + S1 + CH + k + w + S0 + MAJ

1208

1209

ROTATE_ARGS

1210

	movdqa	XTMP3, XTMP2	; XTMP3 = W[-2] {BBAA}

1211

    mov	y0, e		; y0 = e

1212

    mov	y1, a		; y1 = a

1213

    ror	y0, (25-11)	; y0 = e >> (25-11)

1214

	movdqa	XTMP4, XTMP2	; XTMP4 = W[-2] {BBAA}

1215

    xor	y0, e		; y0 = e ^ (e >> (25-11))

1216

    ror	y1, (22-13)	; y1 = a >> (22-13)

1217

    mov	y2, f		; y2 = f

1218

    xor	y1, a		; y1 = a ^ (a >> (22-13)

1219

    ror	y0, (11-6)	; y0 = (e >> (11-6)) ^ (e >> (25-6))

1220

	psrlq	XTMP2, 17	; XTMP2 = W[-2] ror 17 {xBxA}

1221

    xor	y2, g		; y2 = f^g

1222

	psrlq	XTMP3, 19	; XTMP3 = W[-2] ror 19 {xBxA}

1223

    xor	y0, e		; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))

1224

    and	y2, e		; y2 = (f^g)&e

1225

	psrld	XTMP4, 10	; XTMP4 = W[-2] >> 10 {BBAA}

1226

    ror	y1, (13-2)	; y1 = (a >> (13-2)) ^ (a >> (22-2))

1227

    xor	y1, a		; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))

1228

    xor	y2, g		; y2 = CH = ((f^g)&e)^g

1229

    ror	y0, 6		; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)

1230

	pxor	XTMP2, XTMP3

1231

    add	y2, y0		; y2 = S1 + CH

1232

    ror	y1, 2		; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)

1233

    add	y2, [rsp + _XFER + 2*4]	; y2 = k + w + S1 + CH

1234

	pxor	XTMP4, XTMP2	; XTMP4 = s1 {xBxA}

1235

    mov	y0, a		; y0 = a

1236

    add	h, y2		; h = h + S1 + CH + k + w

1237

    mov	y2, a		; y2 = a

1238

	pshufb	XTMP4, SHUF_00BA	; XTMP4 = s1 {00BA}

1239

    or	y0, c		; y0 = a|c

1240

    add	d, h		; d = d + h + S1 + CH + k + w

1241

    and	y2, c		; y2 = a&c

1242

	paddd	XTMP0, XTMP4	; XTMP0 = {..., ..., W[1], W[0]}

1243

    and	y0, b		; y0 = (a|c)&b

1244

    add	h, y1		; h = h + S1 + CH + k + w + S0

1245

	;; compute high s1

1246

	pshufd	XTMP2, XTMP0, 01010000b	; XTMP2 = W[-2] {DDCC}

1247

    or	y0, y2		; y0 = MAJ = (a|c)&b)|(a&c)

1248

    add	h, y0		; h = h + S1 + CH + k + w + S0 + MAJ

1249

1250

ROTATE_ARGS

1251

	movdqa	XTMP3, XTMP2	; XTMP3 = W[-2] {DDCC}

1252

    mov	y0, e		; y0 = e

1253

    ror	y0, (25-11)	; y0 = e >> (25-11)

1254

    mov	y1, a		; y1 = a

1255

	movdqa	X0,    XTMP2	; X0    = W[-2] {DDCC}

1256

    ror	y1, (22-13)	; y1 = a >> (22-13)

1257

    xor	y0, e		; y0 = e ^ (e >> (25-11))

1258

    mov	y2, f		; y2 = f

1259

    ror	y0, (11-6)	; y0 = (e >> (11-6)) ^ (e >> (25-6))

1260

	psrlq	XTMP2, 17	; XTMP2 = W[-2] ror 17 {xDxC}

1261

    xor	y1, a		; y1 = a ^ (a >> (22-13)

1262

    xor	y2, g		; y2 = f^g

1263

	psrlq	XTMP3, 19	; XTMP3 = W[-2] ror 19 {xDxC}

1264

    xor	y0, e		; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))

1265

    and	y2, e		; y2 = (f^g)&e

1266

    ror	y1, (13-2)	; y1 = (a >> (13-2)) ^ (a >> (22-2))

1267

	psrld	X0,    10	; X0 = W[-2] >> 10 {DDCC}

1268

    xor	y1, a		; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))

1269

    ror	y0, 6		; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)

1270

    xor	y2, g		; y2 = CH = ((f^g)&e)^g

1271

	pxor	XTMP2, XTMP3

1272

    ror	y1, 2		; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)

1273

    add	y2, y0		; y2 = S1 + CH

1274

    add	y2, [rsp + _XFER + 3*4]	; y2 = k + w + S1 + CH

1275

	pxor	X0, XTMP2	; X0 = s1 {xDxC}

1276

    mov	y0, a		; y0 = a

1277

    add	h, y2		; h = h + S1 + CH + k + w

1278

    mov	y2, a		; y2 = a

1279

	pshufb	X0, SHUF_DC00	; X0 = s1 {DC00}

1280

    or	y0, c		; y0 = a|c

1281

    add	d, h		; d = d + h + S1 + CH + k + w

1282

    and	y2, c		; y2 = a&c

1283

	paddd	X0, XTMP0	; X0 = {W[3], W[2], W[1], W[0]}

1284

    and	y0, b		; y0 = (a|c)&b

1285

    add	h, y1		; h = h + S1 + CH + k + w + S0

1286

    or	y0, y2		; y0 = MAJ = (a|c)&b)|(a&c)

1287

    add	h, y0		; h = h + S1 + CH + k + w + S0 + MAJ

1288

1289

ROTATE_ARGS

1290

rotate_Xs

1291

%endm

1292

1293

;; input is [rsp + _XFER + %1 * 4]

1294

%macro DO_ROUND 1

1295

    mov	y0, e		; y0 = e

1296

    ror	y0, (25-11)	; y0 = e >> (25-11)

1297

    mov	y1, a		; y1 = a

1298

    xor	y0, e		; y0 = e ^ (e >> (25-11))

1299

    ror	y1, (22-13)	; y1 = a >> (22-13)

1300

    mov	y2, f		; y2 = f

1301

    xor	y1, a		; y1 = a ^ (a >> (22-13)

1302

    ror	y0, (11-6)	; y0 = (e >> (11-6)) ^ (e >> (25-6))

1303

    xor	y2, g		; y2 = f^g

1304

    xor	y0, e		; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))

1305

    ror	y1, (13-2)	; y1 = (a >> (13-2)) ^ (a >> (22-2))

1306

    and	y2, e		; y2 = (f^g)&e

1307

    xor	y1, a		; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))

1308

    ror	y0, 6		; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)

1309

    xor	y2, g		; y2 = CH = ((f^g)&e)^g

1310

    add	y2, y0		; y2 = S1 + CH

1311

    ror	y1, 2		; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)

1312

    add	y2, [rsp + _XFER + %1 * 4]	; y2 = k + w + S1 + CH

1313

    mov	y0, a		; y0 = a

1314

    add	h, y2		; h = h + S1 + CH + k + w

1315

    mov	y2, a		; y2 = a

1316

    or	y0, c		; y0 = a|c

1317

    add	d, h		; d = d + h + S1 + CH + k + w

1318

    and	y2, c		; y2 = a&c

1319

    and	y0, b		; y0 = (a|c)&b

1320

    add	h, y1		; h = h + S1 + CH + k + w + S0

1321

    or	y0, y2		; y0 = MAJ = (a|c)&b)|(a&c)

1322

    add	h, y0		; h = h + S1 + CH + k + w + S0 + MAJ

1323

    ROTATE_ARGS

1324

%endm

1325

1326

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

1327

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

1328

;; void sha256_sse4(void *input_data, UINT32 digest[8], UINT64 num_blks)

1329

;; arg 1 : pointer to input data

1330

;; arg 2 : pointer to digest

1331

;; arg 3 : Num blocks

1332

section .text

1333

global sha256_sse4

1334

align 32

1335

sha256_sse4:

1336

    push	rbx

1337

%ifndef LINUX

1338

    push	rsi

1339

    push	rdi

1340

%endif

1341

    push	rbp

1342

    push	r13

1343

    push	r14

1344

    push	r15

1345

1346

    sub	rsp,STACK_SIZE

1347

%ifndef LINUX

1348

    movdqa	[rsp + _XMM_SAVE + 0*16],xmm6

1349

    movdqa	[rsp + _XMM_SAVE + 1*16],xmm7

1350

    movdqa	[rsp + _XMM_SAVE + 2*16],xmm8

1351

    movdqa	[rsp + _XMM_SAVE + 3*16],xmm9

1352

    movdqa	[rsp + _XMM_SAVE + 4*16],xmm10

1353

    movdqa	[rsp + _XMM_SAVE + 5*16],xmm11

1354

    movdqa	[rsp + _XMM_SAVE + 6*16],xmm12

1355

%endif

1356

1357

    shl	NUM_BLKS, 6	; convert to bytes

1358

    jz	done_hash

1359

    add	NUM_BLKS, INP	; pointer to end of data

1360

    mov	[rsp + _INP_END], NUM_BLKS

1361

1362

    ;; load initial digest

1363

    mov	a,[4*0 + CTX]

1364

    mov	b,[4*1 + CTX]

1365

    mov	c,[4*2 + CTX]

1366

    mov	d,[4*3 + CTX]

1367

    mov	e,[4*4 + CTX]

1368

    mov	f,[4*5 + CTX]

1369

    mov	g,[4*6 + CTX]

1370

    mov	h,[4*7 + CTX]

1371

1372

    movdqa	BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip]

1373

    movdqa	SHUF_00BA, [_SHUF_00BA wrt rip]

1374

    movdqa	SHUF_DC00, [_SHUF_DC00 wrt rip]

1375

1376

loop0:

1377

    lea	TBL,[K256 wrt rip]

1378

1379

    ;; byte swap first 16 dwords

1380

    COPY_XMM_AND_BSWAP	X0, [INP + 0*16], BYTE_FLIP_MASK

1381

    COPY_XMM_AND_BSWAP	X1, [INP + 1*16], BYTE_FLIP_MASK

1382

    COPY_XMM_AND_BSWAP	X2, [INP + 2*16], BYTE_FLIP_MASK

1383

    COPY_XMM_AND_BSWAP	X3, [INP + 3*16], BYTE_FLIP_MASK

1384

1385

    mov	[rsp + _INP], INP

1386

1387

    ;; schedule 48 input dwords, by doing 3 rounds of 16 each

1388

    mov	SRND, 3

1389

align 16

1390

loop1:

1391

    movdqa	XFER, [TBL + 0*16]

1392

    paddd	XFER, X0

1393

    movdqa	[rsp + _XFER], XFER

1394

    FOUR_ROUNDS_AND_SCHED

1395

1396

    movdqa	XFER, [TBL + 1*16]

1397

    paddd	XFER, X0

1398

    movdqa	[rsp + _XFER], XFER

1399

    FOUR_ROUNDS_AND_SCHED

1400

1401

    movdqa	XFER, [TBL + 2*16]

1402

    paddd	XFER, X0

1403

    movdqa	[rsp + _XFER], XFER

1404

    FOUR_ROUNDS_AND_SCHED

1405

1406

    movdqa	XFER, [TBL + 3*16]

1407

    paddd	XFER, X0

1408

    movdqa	[rsp + _XFER], XFER

1409

    add	TBL, 4*16

1410

    FOUR_ROUNDS_AND_SCHED

1411

1412

    sub	SRND, 1

1413

    jne	loop1

1414

1415

    mov	SRND, 2

1416

loop2:

1417

    paddd	X0, [TBL + 0*16]

1418

    movdqa	[rsp + _XFER], X0

1419

    DO_ROUND	0

1420

    DO_ROUND	1

1421

    DO_ROUND	2

1422

    DO_ROUND	3

1423

    paddd	X1, [TBL + 1*16]

1424

    movdqa	[rsp + _XFER], X1

1425

    add	TBL, 2*16

1426

    DO_ROUND	0

1427

    DO_ROUND	1

1428

    DO_ROUND	2

1429

    DO_ROUND	3

1430

1431

    movdqa	X0, X2

1432

    movdqa	X1, X3

1433

1434

    sub	SRND, 1

1435

    jne	loop2

1436

1437

    addm	[4*0 + CTX],a

1438

    addm	[4*1 + CTX],b

1439

    addm	[4*2 + CTX],c

1440

    addm	[4*3 + CTX],d

1441

    addm	[4*4 + CTX],e

1442

    addm	[4*5 + CTX],f

1443

    addm	[4*6 + CTX],g

1444

    addm	[4*7 + CTX],h

1445

1446

    mov	INP, [rsp + _INP]

1447

    add	INP, 64

1448

    cmp	INP, [rsp + _INP_END]

1449

    jne	loop0

1450

1451

done_hash:

1452

%ifndef LINUX

1453

    movdqa	xmm6,[rsp + _XMM_SAVE + 0*16]

1454

    movdqa	xmm7,[rsp + _XMM_SAVE + 1*16]

1455

    movdqa	xmm8,[rsp + _XMM_SAVE + 2*16]

1456

    movdqa	xmm9,[rsp + _XMM_SAVE + 3*16]

1457

    movdqa	xmm10,[rsp + _XMM_SAVE + 4*16]

1458

    movdqa	xmm11,[rsp + _XMM_SAVE + 5*16]

1459

    movdqa	xmm12,[rsp + _XMM_SAVE + 6*16]

1460

%endif

1461

1462

    add	rsp, STACK_SIZE

1463

1464

    pop	r15

1465

    pop	r14

1466

    pop	r13

1467

    pop	rbp

1468

%ifndef LINUX

1469

    pop	rdi

1470

    pop	rsi

1471

%endif

1472

    pop	rbx

1473

1474

ret

1475

1476

1477

section .data

1478

align 64

1479

K256:

1480

    dd	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5

1481

    dd	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5

1482

    dd	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3

1483

    dd	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174

1484

    dd	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc

1485

    dd	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da

1486

    dd	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7

1487

    dd	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967

1488

    dd	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13

1489

    dd	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85

1490

    dd	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3

1491

    dd	0xd192e819,0xd6990624,0xf40e3585,0x106aa070

1492

    dd	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5

1493

    dd	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3

1494

    dd	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208

1495

    dd	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2

1496

1497

PSHUFFLE_BYTE_FLIP_MASK: ddq 0x0c0d0e0f08090a0b0405060700010203

1498

1499

; shuffle xBxA -> 00BA

1500

_SHUF_00BA:              ddq 0xFFFFFFFFFFFFFFFF0b0a090803020100

1501

1502

; shuffle xDxC -> DC00

1503

_SHUF_DC00:              ddq 0x0b0a090803020100FFFFFFFFFFFFFFFF

1504

*/

1505

1506

#endif