MD5 Calculation for a file with buffer.

breeze4me · Post by **breeze4me** » Thu Jun 15, 2006 7:52 am

Code updated for 5.20+

For Windows.

Referred to:
http://etud.epita.fr/~bevand_m/papers/md5-amd64.html

I'm not good at English, so not commented. Sorry.

Not used stack memory but global, because stack seems to be slightly slow.

Code: Select all

;
; MD5 Calculation for a file with buffer.
;
; - Author: breeze4me
; - Version: 0.5
; - June, 2006

DisableDebugger
EnableASM

; // MD5 context.
!Struc MD5_CTX
!{
!  .state rd 4
!  .buffer rb 128
!  .bitsize rd 2
!  .padLen rd 1
!  .blocks rd 1
!  .readblocks rd 1
!  .pbuf rd 1
!  .remainedbytes rd 1
!  .lastBlocks rd 1
!}
Structure MD5_CTX
  state.l[4]    ; state (ABCD)
  buffer.b[128] ; buffer for MD5Final
  bitsize.q
  padLen.l
  blocks.l
  readblocks.l
  pbuf.l
  remainedbytes.l
  lastBlocks.l
EndStructure

#BufferSize=8192 ;128*64. (BufferSize must be a multiple of 64.)
#BlocksPerBuffer= #BufferSize/64

;{ Constants for MD5Transform routine.
#S11 = 7
#S12 = 12
#S13 = 17
#S14 = 22

#S21 = 5
#S22 = 9
#S23 = 14
#S24 = 20

#S31 = 4
#S32 = 11
#S33 = 16
#S34 = 23

#S41 = 6
#S42 = 10
#S43 = 15
#S44 = 21
;}

Global *ctx.MD5_CTX = ?ctx

Macro FF(a, b, c, d, x, s, ac)
  ;(((c ! d) & b) ! d)
  MOV esi, c
  XOr esi, d
  And esi, b
  XOr esi, d
 
  ADD a, esi
  ADD a, dword [x]
  ADD a, ac
  ROL a, s
  ADD a, b
;   a + (F(b, c, d) + x + ac)
;   ROTATE_LEFT(a, s)
;   a + b
EndMacro

Macro GG(a, b, c, d, x, s, ac)
  ;(((b ! c) & d) ! c)
  MOV esi, b
  XOr esi, c
  And esi, d
  XOr esi, c
 
  ADD a, esi
  ADD a, dword [x]
  ADD a, ac
  ROL a, s
  ADD a, b
;   a + (G(b, c, d) + x + ac)
;   ROTATE_LEFT(a, s)
;   a + b
EndMacro

Macro HH(a, b, c, d, x, s, ac)
  ;(b ! c ! d)
  MOV esi, b
  XOr esi, c
  XOr esi, d

  ADD a, esi
  ADD a, dword [x]
  ADD a, ac
  ROL a, s
  ADD a, b
;   a + (H(b, c, d) + x + ac)
;   ROTATE_LEFT(a, s)
;   a + b
EndMacro

Macro II(a, b, c, d, x, s, ac)
  ;(((~d) | b) ! c)
  MOV esi, d
  Not esi
  Or esi, b
  XOr esi, c
 
  ADD a, esi
  ADD a, dword [x]
  ADD a, ac
  ROL a, s
  ADD a, b
;   a + (I(b, c, d) + x + ac)
;   ROTATE_LEFT(a, s)
;   a + b
EndMacro

;MD5 basic transformation. Transforms state based on block.
Procedure MD5Transform()
  !MOV esi, ctx.state
  MOV eax, dword [esi]
  MOV ebx, dword [esi+4]
  MOV ecx, dword [esi+8]
  MOV edx, dword [esi+12]
 
  ;  // Round 1
  FF(eax, ebx, ecx, edx, edi, #S11, $d76aa478); // 1
  FF(edx, eax, ebx, ecx, edi+4, #S12, $e8c7b756); // 2
  FF(ecx, edx, eax, ebx, edi+8, #S13, $242070db); // 3
  FF(ebx, ecx, edx, eax, edi+12, #S14, $c1bdceee); // 4
  FF(eax, ebx, ecx, edx, edi+16, #S11, $f57c0faf); // 5
  FF(edx, eax, ebx, ecx, edi+20, #S12, $4787c62a); // 6
  FF(ecx, edx, eax, ebx, edi+24, #S13, $a8304613); // 7
  FF(ebx, ecx, edx, eax, edi+28, #S14, $fd469501); // 8
  FF(eax, ebx, ecx, edx, edi+32, #S11, $698098d8); // 9
  FF(edx, eax, ebx, ecx, edi+36, #S12, $8b44f7af); // 10
  FF(ecx, edx, eax, ebx, edi+40, #S13, $ffff5bb1); // 11
  FF(ebx, ecx, edx, eax, edi+44, #S14, $895cd7be); // 12
  FF(eax, ebx, ecx, edx, edi+48, #S11, $6b901122); // 13
  FF(edx, eax, ebx, ecx, edi+52, #S12, $fd987193); // 14
  FF(ecx, edx, eax, ebx, edi+56, #S13, $a679438e); // 15
  FF(ebx, ecx, edx, eax, edi+60, #S14, $49b40821); // 16
 
  ;  // Round 2
  GG(eax, ebx, ecx, edx, edi+ 4, #S21, $f61e2562); // 17
  GG(edx, eax, ebx, ecx, edi+24, #S22, $c040b340); // 18
  GG(ecx, edx, eax, ebx, edi+44, #S23, $265e5a51); // 19
  GG(ebx, ecx, edx, eax, edi   , #S24, $e9b6c7aa); // 20
  GG(eax, ebx, ecx, edx, edi+20, #S21, $d62f105d); // 21
  GG(edx, eax, ebx, ecx, edi+40, #S22,  $2441453); // 22
  GG(ecx, edx, eax, ebx, edi+60, #S23, $d8a1e681); // 23
  GG(ebx, ecx, edx, eax, edi+16, #S24, $e7d3fbc8); // 24
  GG(eax, ebx, ecx, edx, edi+36, #S21, $21e1cde6); // 25
  GG(edx, eax, ebx, ecx, edi+56, #S22, $c33707d6); // 26
  GG(ecx, edx, eax, ebx, edi+12, #S23, $f4d50d87); // 27
  GG(ebx, ecx, edx, eax, edi+32, #S24, $455a14ed); // 28
  GG(eax, ebx, ecx, edx, edi+52, #S21, $a9e3e905); // 29
  GG(edx, eax, ebx, ecx, edi+ 8, #S22, $fcefa3f8); // 30
  GG(ecx, edx, eax, ebx, edi+28, #S23, $676f02d9); // 31
  GG(ebx, ecx, edx, eax, edi+48, #S24, $8d2a4c8a); // 32
 
  ;  // Round 3
  HH(eax, ebx, ecx, edx, edi+20, #S31, $fffa3942); // 33
  HH(edx, eax, ebx, ecx, edi+32, #S32, $8771f681); // 34
  HH(ecx, edx, eax, ebx, edi+44, #S33, $6d9d6122); // 35
  HH(ebx, ecx, edx, eax, edi+56, #S34, $fde5380c); // 36
  HH(eax, ebx, ecx, edx, edi+4, #S31, $a4beea44); // 37
  HH(edx, eax, ebx, ecx, edi+16, #S32, $4bdecfa9); // 38
  HH(ecx, edx, eax, ebx, edi+28, #S33, $f6bb4b60); // 39
  HH(ebx, ecx, edx, eax, edi+40, #S34, $bebfbc70); // 40
  HH(eax, ebx, ecx, edx, edi+52, #S31, $289b7ec6); // 41
  HH(edx, eax, ebx, ecx, edi, #S32, $eaa127fa); // 42
  HH(ecx, edx, eax, ebx, edi+12, #S33, $d4ef3085); // 43
  HH(ebx, ecx, edx, eax, edi+24, #S34,  $4881d05); // 44
  HH(eax, ebx, ecx, edx, edi+36, #S31, $d9d4d039); // 45
  HH(edx, eax, ebx, ecx, edi+48, #S32, $e6db99e5); // 46
  HH(ecx, edx, eax, ebx, edi+60, #S33, $1fa27cf8); // 47
  HH(ebx, ecx, edx, eax, edi+8, #S34, $c4ac5665); // 48
 
  ;  // Round 4
  II(eax, ebx, ecx, edx, edi, #S41, $f4292244); // 49
  II(edx, eax, ebx, ecx, edi+28, #S42, $432aff97); // 50
  II(ecx, edx, eax, ebx, edi+56, #S43, $ab9423a7); // 51
  II(ebx, ecx, edx, eax, edi+20, #S44, $fc93a039); // 52
  II(eax, ebx, ecx, edx, edi+48, #S41, $655b59c3); // 53
  II(edx, eax, ebx, ecx, edi+12, #S42, $8f0ccc92); // 54
  II(ecx, edx, eax, ebx, edi+40, #S43, $ffeff47d); // 55
  II(ebx, ecx, edx, eax, edi+4, #S44, $85845dd1); // 56
  II(eax, ebx, ecx, edx, edi+32, #S41, $6fa87e4f); // 57
  II(edx, eax, ebx, ecx, edi+60, #S42, $fe2ce6e0); // 58
  II(ecx, edx, eax, ebx, edi+24, #S43, $a3014314); // 59
  II(ebx, ecx, edx, eax, edi+52, #S44, $4e0811a1); // 60
  II(eax, ebx, ecx, edx, edi+16, #S41, $f7537e82); // 61
  II(edx, eax, ebx, ecx, edi+44, #S42, $bd3af235); // 62
  II(ecx, edx, eax, ebx, edi+8, #S43, $2ad7d2bb); // 63
  II(ebx, ecx, edx, eax, edi+36, #S44, $eb86d391); // 64
 
  !MOV esi, ctx.state
  ADD dword [esi], eax
  ADD dword [esi+4], ebx
  ADD dword [esi+8], ecx
  ADD dword [esi+12], edx
   
EndProcedure

Procedure MD5Init(*ctx.MD5_CTX, totalsize.q, *buf)
  ;Load magic initialization constants.
  *ctx\state[0] = $67452301
  *ctx\state[1] = $efcdab89
  *ctx\state[2] = $98badcfe
  *ctx\state[3] = $10325476
 
  *ctx\bitsize = totalsize << 3
  totalsize+8
 
  *ctx\padLen = 63-(totalsize & %111111)
  *ctx\blocks = ((totalsize + *ctx\padLen + 1) >> 6)
 
  *ctx\readblocks = 0
  *ctx\pbuf = *buf
  totalsize-8
 
  If *ctx\padLen>55
    *ctx\lastBlocks=2
  Else
    *ctx\lastBlocks=1
  EndIf
  *ctx\remainedbytes = totalsize - ((*ctx\blocks-*ctx\lastBlocks) << 6)
  *ctx\blocks - *ctx\lastBlocks
EndProcedure

Procedure MD5Update()
  !MOV edi, dword [ctx.pbuf]
  XOr ecx, ecx
!@@:
  PUSH ecx
  ;PUSH edi
  MD5Transform()
  ;POP edi
 
  ADD edi, 64
  !INC dword [ctx.readblocks]
 
  POP ecx
  INC ecx
  CMP ecx, #BlocksPerBuffer
  !jl @b
EndProcedure

Procedure MD5Final()
  CLD
  !MOV edi, dword [ctx.pbuf]
  XOr ecx, ecx
!@@:
  !MOV ebx, dword [ctx.readblocks]
  !CMP ebx, dword [ctx.blocks]
  !JGE .n
 
  PUSH ecx
  ;PUSH edi
  MD5Transform()
  ;POP edi
 
  ADD edi, 64
  !INC dword [ctx.readblocks]

  POP ecx
  INC ecx
 
  CMP ecx, #BlocksPerBuffer
  !jle @b
  ProcedureReturn
 
!.n:
  MOV esi, edi
  !MOV edi, ctx.buffer
  !MOV ecx, dword [ctx.remainedbytes]
  !REP MOVSB
 
  ;pad bytes
  MOV byte [edi], $80
  INC edi
  XOr eax, eax
  MOV ecx, dword [ctx.padLen]
  !REP STOSB
 
  ;copy size
  !MOV eax, dword [ctx.bitsize]
  !MOV ebx, dword [ctx.bitsize+4]
  MOV dword [edi], eax
  MOV dword [edi+4], ebx
 
  !mov eax, dword [ctx.lastBlocks]
  MOV edi, ctx.buffer
!.lp:
  PUSH eax
  ;PUSH edi
  MD5Transform()
  ;POP edi
 
  ADD edi, 64
  POP eax
  DEC eax
  CMP eax, 0
  !JNE .lp
EndProcedure

Procedure.s MD5str()
  !MOV esi, ctx.state
  MOV eax, [esi]
  MOV ebx, [esi+4]
  MOV ecx, [esi+8]
  MOV edx, [esi+12]
  BSWAP eax
  BSWAP ebx
  BSWAP ecx
  BSWAP edx
  MOV [esi], edx
  MOV [esi+4], ecx
  MOV [esi+8], ebx
  MOV [esi+12], eax
  tmp$= RSet(Hex(PeekQ(@*ctx\state[2])), 16, "0")+RSet(Hex(PeekQ(@*ctx\state[0])), 16, "0")
  ProcedureReturn tmp$
EndProcedure

DisableASM

;--------------
;- Test code 1
;--------------

Restart:

n$=OpenFileRequester("","","*.*",0)
hFile=ReadFile(0, n$)
If hFile
  buf=AllocateMemory(#BufferSize)
  fs.q=Lof(0)
  ;Debug fs
 
  MD5Init(*ctx, fs, buf)
  rs.q=0
 
  Repeat
    len=ReadData(0, buf, #BufferSize)
    ;Debug "read=" + Str(len)
    rs+len
   
    If (len < #BufferSize) Or (rs = fs)
      MD5Final()
      Break
    Else
      MD5Update()
    EndIf
  ForEver
 
  tmp$=MD5str()   ;a bit faster
;   tmp$=""
;   For i=0 To 15
;     tmp$= tmp$ + RSet(Hex(PeekB(*ctx+i) & $FF), 2, "0")
;   Next
 
  CloseFile(0)
 
  If MessageRequester(n$, tmp$ + #CRLF$ + "Quit ?", #MB_YESNO)=#IDNO
    Goto Restart
  EndIf
EndIf

FreeMemory(buf)

;--------------
;- Test code 2
;--------------

buf=AllocateMemory(64)
tmp$="abcdefghij"
len=Len(tmp$)
PokeS(buf, tmp$)

t1=timeGetTime_()
  For i=0 To 100000
    MD5Init(*ctx, len, buf)
    MD5Final()
    md5_1$=MD5str()
  Next
t1=timeGetTime_()-t1

t2=timeGetTime_()
  For i=0 To 100000
    md5_2$=MD5Fingerprint(buf,len)
  Next
t2=timeGetTime_()-t2

EnableDebugger

Debug t1
Debug md5_1$

Debug t2
Debug md5_2$


End

DataSection
ctx:
!ctx MD5_CTX
EndDataSection

Phoenix · Post by **Phoenix** » Thu Jun 15, 2006 11:25 am

What's wrong with PureBasic's MD5FileFingerprint command??????
http://www.purebasic.com/documentation/ ... print.html

breeze4me · Post by **breeze4me** » Thu Jun 15, 2006 11:33 am

Phoenix wrote:What's wrong with PureBasic's MD5FileFingerprint command??????
http://www.purebasic.com/documentation/ ... print.html

For the buffered file manipulation.

I maked a program to calculate big files' checksums.
In the case of using PureBasic's one, it can't cancel the process.
So made this one.

thefool · Post by **thefool** » Thu Jun 15, 2006 11:47 am

Phoenix wrote:What's wrong with PureBasic's MD5FileFingerprint command??????
http://www.purebasic.com/documentation/ ... print.html

If you look at it, isnt Breeze4me's function much faster too?

edit: In all cases on my computer its 4 times as fast! You dont think that counts if you have a program that does these calcs. on a lot of stuff?

Phoenix · Post by **Phoenix** » Thu Jun 15, 2006 12:32 pm

thefool wrote:If you look at it, isnt Breeze4me's function much faster too?

Don't know..... I program in Basic, not Assembly...... If his is faster, then yeah, go for it...... But I would have assumed that PureBasic's command would have been fast too?????? Is it really 4 times slower??????

thefool · Post by **thefool** » Thu Jun 15, 2006 12:40 pm

Yes its 4 times slower. You cannot assume pb's commands are fast. the math lib is actually quite slow in some things and so on.

This code IS really 4 times faster. (And i dont say that just looking at the code, i tried it. WITH debugger off, of course)

Phoenix · Post by **Phoenix** » Thu Jun 15, 2006 12:45 pm

I just tried the code but I get a syntax error when running it.......

Code: Select all

Procedure MD5Transform()
  !MOV esi, ctx.state
  MOV eax, dword [esi] <---- SYNTAX ERROR

gnozal · Post by **gnozal** » Thu Jun 15, 2006 12:48 pm

Did you enable in-line ASM ?

Phoenix · Post by **Phoenix** » Thu Jun 15, 2006 12:57 pm

Thank you, now it works....

mskuma · Post by **mskuma** » Thu Jun 15, 2006 1:11 pm

breeze4me, impressive stuff - thanks alot. This is what I love about this forum.. lots of geniuses here we can learn from. I agree about the speed up (get about 3x improvement).

MikeB · Post by **MikeB** » Thu Jun 15, 2006 2:00 pm

Don't know if it is of any use to anybody, but I wrote a routine to save a bunch of passwords protected by one password so I would only have to remember the one. It has the advantage that I can understand it where these others are totally unintelligible to me. Obviously this little demo has to be modified to use in practice.

Code: Select all

EnableExplicit
Define .s string$, code$, outstring$, numstring$
Define .l k, codelen, codecount, FileLen
Define .b CodeByte
string$ = "The line to be encoded is this, anything would do.   "+Chr(13)+Chr(13)+"      ~~ This is a test of my encryption. ~~ " 
MessageRequester("This is what will be saved",string$)
code$ = Trim(InputRequester("If you forget this key you can't get the file back!","Enter your Code Key",""))
outstring$ = ""
codelen = Len(code$)
codecount = 1
DeleteFile("F:\MYTEMP\Temp\PBcodeTest")  ; delete a previous (if any) so it won't be longer than needed
If OpenFile(0,"F:\MYTEMP\Temp\PBcodeTest")
	WriteLong(0,Len(string$))
	For k = 1 To Len(string$)
		CodeByte = Asc(Mid(string$,k,1)) ! Asc(Mid(code$,codecount,1))
		WriteByte(0,CodeByte)
		numstring$ = numstring$ + Str(CodeByte)+", "
		codecount+1
		If codecount > codelen
			codecount = 1
		EndIf
	Next
	CloseFile(0)
EndIf
MessageRequester("This is what is saved. (Without the , 's)",numstring$)
;
;   Normally the second part will be called before the first.  Read the file - decode - do something - recode - save it.
;
;  Now get it back
codecount = 1
code$ = Trim(InputRequester("Code Key, must be the same as saved with!","Enter your Code Key",""))
codelen = Len(code$)
If OpenFile(0,"F:\MYTEMP\Temp\PBcodeTest")
	FileLen = ReadLong(0)
	For k = 1 To FileLen
		outstring$ = outstring$+Chr(ReadByte(0) ! Asc(Mid(code$,codecount,1)))
		codecount+1
		If codecount > codelen
			codecount = 1
		EndIf
	Next
	CloseFile(0)
EndIf
MessageRequester("Recovered Line", outstring$)

vkleonid · Post by **vkleonid** » Thu Jun 15, 2006 11:11 pm

breeze4me

Error when running it

:

thefool · Post by **thefool** » Thu Jun 15, 2006 11:33 pm

Hi!

What version of pb do you use?

Inf0Byt3 · Post by **Inf0Byt3** » Fri Jun 16, 2006 12:12 am

Very nice work! Thanks.

breeze4me · Post by **breeze4me** » Fri Jun 16, 2006 2:39 am

vkleonid wrote: Error when running it :

This code is for PB v4.
If you use it, check the jaPBe's option(Enable inline asm support).

PureBasic Forums - English

MD5 Calculation for a file with buffer.

MD5 Calculation for a file with buffer.

Re: MD5 Calculation for a file with buffer.

Re: MD5 Calculation for a file with buffer.

Re: MD5 Calculation for a file with buffer.

Re: MD5 Calculation for a file with buffer.