MD5 Calculation for a file with buffer.

breeze4me · Post by **breeze4me** » Sat Jun 17, 2006 12:28 pm

much faster way. (about 7~8 times faster)

Global MD5str.s=Space(32) ;faster
;Global MD5str.s{32} ;slower
chrTable.s="0123456789abcdef"

Procedure.s MD5strA() ;for ansi
  ;Protected chrTable.s="0123456789abcdef"
  Shared chrTable
  MOV esi, ctx
  MOV edi, MD5str  ;for normal string
  ;LEA edi, MD5str  ;for fixed string
  
  MOV ecx, 16
!@@:
  XOR eax, eax
  MOV al, byte [esi]
  INC esi
  
  SHL ax, 4
  SHR al, 4
  
  XOR ebx, ebx
  MOV bl, ah
  ADD ebx, chrTable
  MOV bl, byte [ebx]
  MOV byte [edi], bl
  INC edi
  
  XOR ebx, ebx
  MOV bl, al
  ADD ebx, chrTable
  MOV bl, byte [ebx]
  MOV byte [edi], bl
  INC edi
  
;   ;for Debuger
;   DEC ecx
;   CMP ecx,0
;   !JNE @r
  !loop @r  ;without Debuger
  ;ProcedureReturn MD5str
EndProcedure

Procedure.s MD5strW() ;for unicode
  Shared chrTable
  MOV esi, ctx
  MOV edi, MD5str
  
  MOV ecx, 16
!@@:
  XOR eax, eax
  MOV al, byte [esi]
  INC esi
  
  SHL ax, 4
  SHR al, 4
  
  XOR ebx, ebx
  MOV bl, ah
  SHL bx, 1
  ADD ebx, chrTable
  MOV bl, byte [ebx]
  MOV byte [edi], bl
  MOV byte [edi+1], 0
  ADD edi, 2
  
  XOR ebx, ebx
  MOV bl, al
  SHL bx, 1
  ADD ebx, chrTable
  MOV bl, byte [ebx]
  MOV byte [edi], bl
  MOV byte [edi+1], 0
  ADD edi, 2
  
;   ;for Debuger
;   DEC ecx
;   CMP ecx,0
;   !JNE @r
  !loop @r  ;without Debuger
  ;ProcedureReturn MD5str
EndProcedure

Code: Select all

;-------------- 
;- Test code 2 
;-------------- 

buf=AllocateMemory(#BufferSize)
tmp$="abcdefghijklmnopqr"
len=Len(tmp$) 
PokeS(buf, tmp$) 

t1=timeGetTime_() 
  For i=0 To 100000 
    MD5Init(*ctx, len, buf) 
    MD5Final() 
;     MD5strW()
    MD5strA()
    md5_1$=MD5str
  Next 
t1=timeGetTime_()-t1 

t2=timeGetTime_() 
  For i=0 To 100000 
    md5_2$=MD5Fingerprint(buf,len) 
  Next 
t2=timeGetTime_()-t2 

EnableDebugger 

Debug t1 
Debug md5_1$

Debug t2 
Debug md5_2$ 

End

Phoenix · Post by **Phoenix** » Sun Jun 18, 2006 2:03 am

thefool wrote:This code IS really 4 times faster.

Not here...... I tried it with a 67.5 MB file (a music video) and PureBasic's MD5 command did it in 1703 ms but breeze4me's code was taking forever..... so long that I ended up killing the executable from the Task Manager....... here's the code I was using..... if I've done something wrong then can somebody please correct it...... because I'm not impressed so far.....

Code: Select all

!Struc MD5_CTX
!{
!  .state rd 4
!  .buffer rb 128
!  .bitsize rd 2
!  .padLen rd 1
!  .blocks rd 1
!  .readblocks rd 1
!  .pbuf rd 1
!  .remainedbytes rd 1
!  .lastBlocks rd 1
!}
Structure MD5_CTX
  state.l[4]    ; state (ABCD)
  buffer.b[128] ; buffer for MD5Final
  bitsize.q
  padLen.l
  blocks.l
  readblocks.l
  pbuf.l
  remainedbytes.l
  lastBlocks.l
EndStructure

#BufferSize=8192 ;128*64. (BufferSize must be a multiple of 64.)
#BlocksPerBuffer= #BufferSize/64

;{ Constants for MD5Transform routine.
#S11 = 7
#S12 = 12
#S13 = 17
#S14 = 22

#S21 = 5
#S22 = 9
#S23 = 14
#S24 = 20

#S31 = 4
#S32 = 11
#S33 = 16
#S34 = 23

#S41 = 6
#S42 = 10
#S43 = 15
#S44 = 21
;}

Global *ctx.MD5_CTX = ?ctx

Macro FF(a, b, c, d, x, s, ac)
  ;(((c ! d) & b) ! d)
  MOV esi, c
  XOR esi, d
  AND esi, b
  XOR esi, d

  ADD a, esi
  ADD a, dword [x]
  ADD a, ac
  ROL a, s
  ADD a, b
;   a + (F(b, c, d) + x + ac)
;   ROTATE_LEFT(a, s)
;   a + b
EndMacro

Macro GG(a, b, c, d, x, s, ac)
  ;(((b ! c) & d) ! c)
  MOV esi, b
  XOR esi, c
  AND esi, d
  XOR esi, c

  ADD a, esi
  ADD a, dword [x]
  ADD a, ac
  ROL a, s
  ADD a, b
;   a + (G(b, c, d) + x + ac)
;   ROTATE_LEFT(a, s)
;   a + b
EndMacro

Macro HH(a, b, c, d, x, s, ac)
  ;(b ! c ! d)
  MOV esi, b
  XOR esi, c
  XOR esi, d

  ADD a, esi
  ADD a, dword [x]
  ADD a, ac
  ROL a, s
  ADD a, b
;   a + (H(b, c, d) + x + ac)
;   ROTATE_LEFT(a, s)
;   a + b
EndMacro

Macro II(a, b, c, d, x, s, ac)
  ;(((~d) | b) ! c)
  MOV esi, d
  NOT esi
  OR esi, b
  XOR esi, c

  ADD a, esi
  ADD a, dword [x]
  ADD a, ac
  ROL a, s
  ADD a, b
;   a + (I(b, c, d) + x + ac)
;   ROTATE_LEFT(a, s)
;   a + b
EndMacro

;MD5 basic transformation. Transforms state based on block.
Procedure MD5Transform()
  !MOV esi, ctx.state
  MOV eax, dword [esi]
  MOV ebx, dword [esi+4]
  MOV ecx, dword [esi+8]
  MOV edx, dword [esi+12]

  ;  // Round 1
  FF(eax, ebx, ecx, edx, edi, #S11, $d76aa478); // 1
  FF(edx, eax, ebx, ecx, edi+4, #S12, $e8c7b756); // 2
  FF(ecx, edx, eax, ebx, edi+8, #S13, $242070db); // 3
  FF(ebx, ecx, edx, eax, edi+12, #S14, $c1bdceee); // 4
  FF(eax, ebx, ecx, edx, edi+16, #S11, $f57c0faf); // 5
  FF(edx, eax, ebx, ecx, edi+20, #S12, $4787c62a); // 6
  FF(ecx, edx, eax, ebx, edi+24, #S13, $a8304613); // 7
  FF(ebx, ecx, edx, eax, edi+28, #S14, $fd469501); // 8
  FF(eax, ebx, ecx, edx, edi+32, #S11, $698098d8); // 9
  FF(edx, eax, ebx, ecx, edi+36, #S12, $8b44f7af); // 10
  FF(ecx, edx, eax, ebx, edi+40, #S13, $ffff5bb1); // 11
  FF(ebx, ecx, edx, eax, edi+44, #S14, $895cd7be); // 12
  FF(eax, ebx, ecx, edx, edi+48, #S11, $6b901122); // 13
  FF(edx, eax, ebx, ecx, edi+52, #S12, $fd987193); // 14
  FF(ecx, edx, eax, ebx, edi+56, #S13, $a679438e); // 15
  FF(ebx, ecx, edx, eax, edi+60, #S14, $49b40821); // 16

  ;  // Round 2
  GG(eax, ebx, ecx, edx, edi+ 4, #S21, $f61e2562); // 17
  GG(edx, eax, ebx, ecx, edi+24, #S22, $c040b340); // 18
  GG(ecx, edx, eax, ebx, edi+44, #S23, $265e5a51); // 19
  GG(ebx, ecx, edx, eax, edi   , #S24, $e9b6c7aa); // 20
  GG(eax, ebx, ecx, edx, edi+20, #S21, $d62f105d); // 21
  GG(edx, eax, ebx, ecx, edi+40, #S22,  $2441453); // 22
  GG(ecx, edx, eax, ebx, edi+60, #S23, $d8a1e681); // 23
  GG(ebx, ecx, edx, eax, edi+16, #S24, $e7d3fbc8); // 24
  GG(eax, ebx, ecx, edx, edi+36, #S21, $21e1cde6); // 25
  GG(edx, eax, ebx, ecx, edi+56, #S22, $c33707d6); // 26
  GG(ecx, edx, eax, ebx, edi+12, #S23, $f4d50d87); // 27
  GG(ebx, ecx, edx, eax, edi+32, #S24, $455a14ed); // 28
  GG(eax, ebx, ecx, edx, edi+52, #S21, $a9e3e905); // 29
  GG(edx, eax, ebx, ecx, edi+ 8, #S22, $fcefa3f8); // 30
  GG(ecx, edx, eax, ebx, edi+28, #S23, $676f02d9); // 31
  GG(ebx, ecx, edx, eax, edi+48, #S24, $8d2a4c8a); // 32

  ;  // Round 3
  HH(eax, ebx, ecx, edx, edi+20, #S31, $fffa3942); // 33
  HH(edx, eax, ebx, ecx, edi+32, #S32, $8771f681); // 34
  HH(ecx, edx, eax, ebx, edi+44, #S33, $6d9d6122); // 35
  HH(ebx, ecx, edx, eax, edi+56, #S34, $fde5380c); // 36
  HH(eax, ebx, ecx, edx, edi+4, #S31, $a4beea44); // 37
  HH(edx, eax, ebx, ecx, edi+16, #S32, $4bdecfa9); // 38
  HH(ecx, edx, eax, ebx, edi+28, #S33, $f6bb4b60); // 39
  HH(ebx, ecx, edx, eax, edi+40, #S34, $bebfbc70); // 40
  HH(eax, ebx, ecx, edx, edi+52, #S31, $289b7ec6); // 41
  HH(edx, eax, ebx, ecx, edi, #S32, $eaa127fa); // 42
  HH(ecx, edx, eax, ebx, edi+12, #S33, $d4ef3085); // 43
  HH(ebx, ecx, edx, eax, edi+24, #S34,  $4881d05); // 44
  HH(eax, ebx, ecx, edx, edi+36, #S31, $d9d4d039); // 45
  HH(edx, eax, ebx, ecx, edi+48, #S32, $e6db99e5); // 46
  HH(ecx, edx, eax, ebx, edi+60, #S33, $1fa27cf8); // 47
  HH(ebx, ecx, edx, eax, edi+8, #S34, $c4ac5665); // 48

  ;  // Round 4
  II(eax, ebx, ecx, edx, edi, #S41, $f4292244); // 49
  II(edx, eax, ebx, ecx, edi+28, #S42, $432aff97); // 50
  II(ecx, edx, eax, ebx, edi+56, #S43, $ab9423a7); // 51
  II(ebx, ecx, edx, eax, edi+20, #S44, $fc93a039); // 52
  II(eax, ebx, ecx, edx, edi+48, #S41, $655b59c3); // 53
  II(edx, eax, ebx, ecx, edi+12, #S42, $8f0ccc92); // 54
  II(ecx, edx, eax, ebx, edi+40, #S43, $ffeff47d); // 55
  II(ebx, ecx, edx, eax, edi+4, #S44, $85845dd1); // 56
  II(eax, ebx, ecx, edx, edi+32, #S41, $6fa87e4f); // 57
  II(edx, eax, ebx, ecx, edi+60, #S42, $fe2ce6e0); // 58
  II(ecx, edx, eax, ebx, edi+24, #S43, $a3014314); // 59
  II(ebx, ecx, edx, eax, edi+52, #S44, $4e0811a1); // 60
  II(eax, ebx, ecx, edx, edi+16, #S41, $f7537e82); // 61
  II(edx, eax, ebx, ecx, edi+44, #S42, $bd3af235); // 62
  II(ecx, edx, eax, ebx, edi+8, #S43, $2ad7d2bb); // 63
  II(ebx, ecx, edx, eax, edi+36, #S44, $eb86d391); // 64

  !MOV esi, ctx.state
  ADD dword [esi], eax
  ADD dword [esi+4], ebx
  ADD dword [esi+8], ecx
  ADD dword [esi+12], edx

EndProcedure

Procedure MD5Init(*ctx.MD5_CTX, totalsize.q, *buf)
  ;Load magic initialization constants.
  *ctx\state[0] = $67452301
  *ctx\state[1] = $efcdab89
  *ctx\state[2] = $98badcfe
  *ctx\state[3] = $10325476

  *ctx\bitsize = totalsize <<3>> 6)

  *ctx\readblocks = 0
  *ctx\pbuf = *buf
  totalsize-8

  If *ctx\padLen>55
    *ctx\lastBlocks=2
  Else
    *ctx\lastBlocks=1
  EndIf
  *ctx\remainedbytes = totalsize - ((*ctx\blocks-*ctx\lastBlocks) << 6)
  *ctx\blocks - *ctx\lastBlocks
EndProcedure

Procedure MD5Update()
  !MOV edi, dword [ctx.pbuf]
  XOR ecx, ecx
!@@:
  PUSH ecx
  ;PUSH edi
  MD5Transform()
  ;POP edi

  ADD edi, 64
  !INC dword [ctx.readblocks]

  POP ecx
  INC ecx
  CMP ecx, #BlocksPerBuffer
  !jl @b
EndProcedure

Procedure MD5Final()
  CLD
  !MOV edi, dword [ctx.pbuf]
  XOR ecx, ecx
!@@:
  !MOV ebx, dword [ctx.readblocks]
  !CMP ebx, dword [ctx.blocks]
  !JGE .n

  PUSH ecx
  ;PUSH edi
  MD5Transform()
  ;POP edi

  ADD edi, 64
  !INC dword [ctx.readblocks]

  POP ecx
  INC ecx

  CMP ecx, #BlocksPerBuffer
  !jle @b
  ProcedureReturn

!.n:
  MOV esi, edi
  !MOV edi, ctx.buffer
  !MOV ecx, dword [ctx.remainedbytes]
  !REP MOVSB

  ;pad bytes
  MOV byte [edi], $80
  INC edi
  XOR eax, eax
  MOV ecx, dword [ctx.padLen]
  !REP STOSB

  ;copy size
  !MOV eax, dword [ctx.bitsize]
  !MOV ebx, dword [ctx.bitsize+4]
  MOV dword [edi], eax
  MOV dword [edi+4], ebx

  !mov eax, dword [ctx.lastBlocks]
  MOV edi, ctx.buffer
!.lp:
  PUSH eax
  ;PUSH edi
  MD5Transform()
  ;POP edi

  ADD edi, 64
  POP eax
  DEC eax
  CMP eax, 0
  !JNE .lp
EndProcedure

Procedure.s MD5str()
  !MOV esi, ctx.state
  MOV eax, [esi]
  MOV ebx, [esi+4]
  MOV ecx, [esi+8]
  MOV edx, [esi+12]
  BSWAP eax
  BSWAP ebx
  BSWAP ecx
  BSWAP edx
  MOV [esi], edx
  MOV [esi+4], ecx
  MOV [esi+8], ebx
  MOV [esi+12], eax
  tmp$= RSet(HexQ(PeekQ(@*ctx\state[2])), 16, "0")+RSet(HexQ(PeekQ(@*ctx\state[0])), 16, "0")
  ProcedureReturn tmp$
EndProcedure

Procedure.s FastMD5FileFingerprint(file$)
  hFile=ReadFile(0, file$)
  If hFile
    buf=AllocateMemory(#BufferSize)
    fs.q=Lof(0)
    ;Debug fs

    MD5Init(*ctx, fs, buf)
    rs.q=0

    Repeat
      len=ReadData(0, buf, #BufferSize)
      ;Debug "read=" + Str(len)
      rs+len

      If (len < #BufferSize) Or (rs = fs)
        MD5Final()
        Break
      Else
        MD5Update()
      EndIf
    ForEver
    tmp$=MD5str()   ;a bit faster
    CloseFile(0)
    ProcedureReturn tmp$
  EndIf
EndProcedure

DisableDebugger

f$="E:\Media\Kylie Minogue - I Believe In You.mpg" ; 67.5 MB

t1=GetTickCount_() : t1$=MD5FileFingerprint(f$) : r1=GetTickCount_()-t1 ; 1703 ms.
t2=GetTickCount_() : t2$=FastMD5FileFingerprint(f$) : r2=GetTickCount_()-t2 ; TAKES FOREVER.

MessageRequester("MD5 for 67.5 MB file","Normal: "+t1$+" / "+Str(r1)+"ms"+#CRLF$+"Fast: "+t2$+" / "+Str(r2)+"ms")

End

DataSection
ctx:
!ctx MD5_CTX
EndDataSection

breeze4me · Post by **breeze4me** » Sun Jun 18, 2006 2:40 am

Phoenix wrote:
thefool wrote:This code IS really 4 times faster.
Not here...... I tried it with a 67.5 MB file (a music video) and PureBasic's MD5 command did it in 1703 ms but breeze4me's code was taking forever..... so long that I ended up killing the executable from the Task Manager....... here's the code I was using..... if I've done something wrong then can somebody please correct it...... because I'm not impressed so far.....

Code: Select all

DisableDebugger ;<-- DisableDebugger must be here, or compile without debugger.

!Struc MD5_CTX
!{
!  .state rd 4
!  .buffer rb 128
!  .bitsize rd 2
!  .padLen rd 1
!  .blocks rd 1
!  .readblocks rd 1
!  .pbuf rd 1
!  .remainedbytes rd 1
!  .lastBlocks rd 1
!}
Structure MD5_CTX
  state.l[4]    ; state (ABCD)
  buffer.b[128] ; buffer for MD5Final
  bitsize.q
  padLen.l
  blocks.l
  readblocks.l
  pbuf.l
  remainedbytes.l
  lastBlocks.l
EndStructure
 ......

I tested it with 700MB file.
it tested 2 times, because of disk cache's influence.

secondary test's result is,

when processing single memory buffered data, it will be faster about 4~8 times.
In this case(the hard drive's file), only about twice faster.

Phoenix · Post by **Phoenix** » Sun Jun 18, 2006 3:49 am

Thanks breeze4me...... that did the trick.....

thefool · Post by **thefool** » Sun Jun 18, 2006 10:29 am

Phoenix wrote:Thanks breeze4me...... that did the trick.....

Was it any faster then

Phoenix · Post by **Phoenix** » Sun Jun 18, 2006 12:00 pm

thefool wrote:Was it any faster then

Yes..... "that did the trick"

Seems that people are coming up with faster variations of native PureBasic commands...... for example this tip and the Plot() one....... I wonder if Fred has spoken to them about using them for future updates?????

Dummy · Post by **Dummy** » Fri Jul 07, 2006 1:48 pm

vkleonid wrote:breeze4me

Error when running it :

still getting this error

even if using pb 4.00

pls fix.

breeze4me · Post by **breeze4me** » Fri Jul 07, 2006 3:19 pm

Dummy wrote: still getting this error

even if using pb 4.00

pls fix.

Code: Select all

; // MD5 context. 
!Struc MD5_CTX 
!{ 
!  .state rd 4 
!  .buffer rb 128   ;<--- jaPBe changed this line into .Buffer...
!  .bitsize rd 2 
!  .padLen rd 1 
!  .blocks rd 1 
!  .readblocks rd 1 
!  .pbuf rd 1 
!  .remainedbytes rd 1 
!  .lastBlocks rd 1 
!}

Change .Buffer into .buffer.
or change all the ".buffer" into any other string, something like ".buffers".

I just found this solution.
It happens only with jaPBe and I'm using only PB IDE, so I didn't know the reason.

Sorry for the inconvenience.

Dummy · Post by **Dummy** » Fri Jul 07, 2006 5:07 pm

thx

I simply changed all calls to ctx.buffer to ctx.Buffer now everything works finde

PB · Post by PB » Mon Dec 18, 2006 9:28 pm

This tip no longer works with PureBasic v4.02, observe:

(Had to use ImageShack because IMGspot wasn't responding).

Dummy · Post by **Dummy** » Tue Dec 19, 2006 12:12 am

PB wrote:This tip no longer works with PureBasic v4.02, observe:

(Had to use ImageShack because IMGspot wasn't responding).

PB related bug. Wait for Fred to fix it.

kinglestat · Post by **kinglestat** » Mon Feb 19, 2007 7:30 pm

I get an error
ROL eax, #S11 undefined

but #S11 is there.....

any ideas ?

cheers

KingLestat

PureBasic Forums - English

MD5 Calculation for a file with buffer.

Re: MD5 Calculation for a file with buffer.

Re: MD5 Calculation for a file with buffer.