It is currently Fri Oct 18, 2019 5:22 am

All times are UTC + 1 hour




Post new topic Reply to topic  [ 19 posts ]  Go to page 1, 2  Next
Author Message
 Post subject: XXH32 Module (xxHash)
PostPosted: Mon May 21, 2018 3:00 pm 
Offline
PureBasic Expert
PureBasic Expert

Joined: Sun Aug 08, 2004 5:21 am
Posts: 3479
Location: Netherlands
XXH32 Module (xxHash)

Using Digest=#True for the XXH32_Update procedure does not affect State.
With the XXH32_Update procedure you can generate an intermediate hash, add more bytes and generate a new hash.

Code:
; XXH32 module by Wilbert

; A PureBasic port of the XXH32 algorithm
; The XXH32 algorithm was created by Yann Collet

; Last code update : May 23, 2018

; Callback for XXH32_FromFile:
; Procedure ProgressCallback(BytesRead.q, FileSize.q, UserData)

DeclareModule XXH32
 
  Structure XXH32_State ; structure size = 40 bytes
    v.l[4]              ; offset 0
    memory.l[4]         ; offset 16
    total_len.q         ; offset 32
  EndStructure 
 
  Declare.s XXH32_Hex(Value.l)
  Declare.l XXH32(*Buffer, Size, Seed=0)
  Declare XXH32_Init(*State, Seed=0)
  Declare.l XXH32_Update(*State, *Buffer, Size, Digest=#False)
  Declare.l XXH32_FromFile(Filename.s, Seed=0, *ProgressCallback=0, UserData=0)
 
EndDeclareModule

Module XXH32
  EnableExplicit
  DisableDebugger
  EnableASM
 
  CompilerIf #PB_Compiler_Processor = #PB_Processor_x86
    Macro rbx : ebx : EndMacro
    Macro rsi : esi : EndMacro
    Macro rdi : edi : EndMacro
    Macro rsp : esp : EndMacro   
    Macro rbp : ebp : EndMacro 
  CompilerEndIf
 
  Macro M_XXH32_Hex(o1, o2)
    !movzx eax, byte [p.v_Value+o1]
    !movzx ecx, byte [p.v_Value+o1+1]
    !shl ecx, 16
    !or eax, ecx
    !mov ecx, eax
    !shl ecx, 4
    !or eax, ecx
    !and eax, 0x0f0f0f0f
    !lea ecx, [eax + 0x06060606]
    !or eax, 0x30303030
    !shr ecx, 4
    !and ecx, 0x01010101
    !imul ecx, 0x27
    !add eax, ecx
    !bswap eax
    !mov [p.v_Result+o2], eax
  EndMacro
 
  Procedure.s XXH32_Hex(Value.l)
    Protected Result.q
    M_XXH32_Hex(0, 4)
    M_XXH32_Hex(2, 0)
    ProcedureReturn PeekS(@Result, 8, #PB_Ascii)
  EndProcedure
 
  Macro M_XXH32_Process(lane)
    imul ebp, [rsi], 0x85ebca77   ; accN = accN + (laneN * prime32_2)
    add rsi, 4
    !add lane, ebp
    !rol lane, 13                 ; accN = accN <<< 13
    !imul lane, 0x9e3779b1        ; accN = accN * prime32_1
  EndMacro
 
  Macro M_XXH32_Final(shift)
    !mov ecx, eax                 ; acc = acc xor (acc >> shift)
    !shr ecx, shift
    !xor eax, ecx
  EndMacro
 
  Procedure.l XXH32(*Buffer, Size, Seed=0)
   
    mov [rsp-8], rsi
    mov [rsp-16], rdi
    mov rsi, [p.p_Buffer]
    mov rdi, [p.v_Size]
    sub rdi, 16
    !jnc .init
   
    ; size < 16
    !mov eax, [p.v_Seed]
    !add eax, 0x165667b1          ; acc = seed + prime32_5
    !jmp .add_size
   
    ; initialize internal accumulators
    !.init:
    mov [rsp-24], rbx
    mov [rsp-32], rbp
    !mov ecx, [p.v_Seed]          ; acc3 = seed
    !lea eax, [ecx + 0x24234428]  ; acc1 = seed + (prime32_1 + prime32_2)
    !lea ebx, [ecx + 0x85ebca77]  ; acc2 = seed + prime32_2
    !lea edx, [ecx + 0x61c8864f]  ; acc4 = seed + (-prime32_1)
   
    ; process stripes
    !.process:
    M_XXH32_Process(eax)
    M_XXH32_Process(ebx)
    M_XXH32_Process(ecx)
    M_XXH32_Process(edx)
    sub rdi, 16
    !jnc .process
   
    ; accumulator convergence
    !rol eax, 1                   ; acc = (acc1 <<< 1) + (acc2 << 7) + (acc3 << 12) + (acc4 << 18)
    !rol ebx, 7
    !add eax, ebx
    !rol ecx, 12
    !add eax, ecx
    !rol edx, 18
    !add eax, edx
    mov rbp, [rsp-32]
    mov rbx, [rsp-24]
   
    ; add size
    !.add_size:
    !add eax, [p.v_Size]
   
    ; consume remaining input
    add rdi, 16
    sub rdi, 4
    !jnc .consume4
    add rdi, 4
    !jz .final_mix
    !jmp .consume1
   
    !.consume4:
    imul ecx, [rsi], 0xc2b2ae3d   ; acc = acc + lane * prime32_3
    add rsi, 4
    !add eax, ecx
    !rol eax, 17                  ; acc = (acc <<< 17) * prime32_4
    !imul eax, 0x27d4eb2f
    sub rdi, 4
    !jnc .consume4
    add rdi, 4
    !jz .final_mix
   
    !.consume1:
    movzx ecx, byte [rsi]         ; acc = acc + lane * prime32_5
    add rsi, 1
    !imul ecx, 0x165667b1
    !add eax, ecx
    !rol eax, 11                  ; acc = (acc <<< 11) * prime32_1
    !imul eax, 0x9e3779b1
    sub rdi, 1
    !jnz .consume1
   
    ; final mix
    !.final_mix:
    M_XXH32_Final(15)             ; acc = acc xor (acc >> 15)
    !imul eax, 0x85ebca77         ; acc = acc * prime32_2
    M_XXH32_Final(13)             ; acc = acc xor (acc >> 13)
    !imul eax, 0xc2b2ae3d         ; acc = acc * prime32_3
    M_XXH32_Final(16)             ; acc = acc xor (acc >> 16)
    mov rdi, [rsp-16] 
    mov rsi, [rsp-8]
    ProcedureReturn
   
  EndProcedure
 
  Procedure.l XXH32_ROL(v.l, n.l)
    !mov eax, [p.v_v]
    !mov ecx, [p.v_n]
    !rol eax, cl
    ProcedureReturn
  EndProcedure
 
  Procedure XXH32_Init(*State.XXH32_State, Seed=0)
    *State\v[0] = Seed + $24234428  ; acc1 = seed + (prime32_1 + prime32_2)
    *State\v[1] = Seed + $85ebca77  ; acc2 = seed + prime32_2
    *State\v[2] = Seed              ; acc3 = seed
    *State\v[3] = Seed + $61c8864f  ; acc4 = seed + (-prime32_1)
    *State\total_len = 0
  EndProcedure
 
  Procedure.l XXH32_Update(*State.XXH32_State, *Buffer, Size, Digest=#False)
    Protected.l acc, i, n, mempos = *State\total_len & 15
    *State\total_len + Size
    If mempos
      If mempos + Size < 16
        CopyMemory(*Buffer, @*State\memory + mempos, Size)
        Size = 0
      Else
        n = 16 - mempos
        CopyMemory(*Buffer, @*State\memory + mempos, n)
        *State\v[0] = XXH32_ROL(*State\v[0] + *State\memory[0] * $85ebca77, 13) * $9e3779b1
        *State\v[1] = XXH32_ROL(*State\v[1] + *State\memory[1] * $85ebca77, 13) * $9e3779b1
        *State\v[2] = XXH32_ROL(*State\v[2] + *State\memory[2] * $85ebca77, 13) * $9e3779b1
        *State\v[3] = XXH32_ROL(*State\v[3] + *State\memory[3] * $85ebca77, 13) * $9e3779b1
        *Buffer + n
        Size - n
      EndIf
    EndIf
    n = Size & 15
    If n
      CopyMemory(*Buffer + Size - n, @*State\memory, n)
      Size - n
    EndIf
    If Size
      ; load state
      mov [rsp-8], rsi
      mov [rsp-16], rdi
      mov [rsp-24], rbx
      mov [rsp-32], rbp
      mov rdi, [p.p_State]
      mov eax, [rdi]
      mov ebx, [rdi+4]
      mov ecx, [rdi+8]
      mov edx, [rdi+12]
      mov rsi, [p.p_Buffer]
      mov rdi, [p.v_Size]
      ; process stripes
      !.process:
      M_XXH32_Process(eax)
      M_XXH32_Process(ebx)
      M_XXH32_Process(ecx)
      M_XXH32_Process(edx)
      sub rdi, 16
      !jnz .process   
      ; save state
      mov rdi, [p.p_State]
      mov [rdi], eax
      mov [rdi+4], ebx
      mov [rdi+8], ecx
      mov [rdi+12], edx
      mov rbp, [rsp-32]
      mov rbx, [rsp-24]
      mov rdi, [rsp-16] 
      mov rsi, [rsp-8]     
    EndIf
    If Digest
      ; accumulator convergence     
      If *State\total_len < 16
        acc = *State\v[2] + $165667b1
      Else
        acc = XXH32_ROL(*State\v[0], 1) + XXH32_ROL(*State\v[1], 7) +
              XXH32_ROL(*State\v[2], 12) + XXH32_ROL(*State\v[3], 18)
      EndIf
      ; add size
      acc + *State\total_len
      ; consume remaining input     
      n = *State\total_len & 15
      While n >= 4
        acc = XXH32_ROL(acc + *State\memory[i] * $c2b2ae3d, 17) * $27d4eb2f
        i + 1
        n - 4
      Wend
      i = *State\memory[i]
      While n
        acc = XXH32_ROL(acc + (i & $ff) * $165667b1, 11) * $9e3779b1
        i >> 8
        n - 1
      Wend
      ; final mix
      !mov eax, [p.v_acc]
      M_XXH32_Final(15)             ; acc = acc xor (acc >> 15)
      !imul eax, 0x85ebca77         ; acc = acc * prime32_2
      M_XXH32_Final(13)             ; acc = acc xor (acc >> 13)
      !imul eax, 0xc2b2ae3d         ; acc = acc * prime32_3
      M_XXH32_Final(16)             ; acc = acc xor (acc >> 16)
      ProcedureReturn
    Else
      ProcedureReturn 0
    EndIf
  EndProcedure
 
  #BufferSize = 65536; 64 KiB buffer
 
  Prototype ProgressCallback(BytesRead.q, FileSize.q, UserData)
 
  Procedure.l XXH32_FromFile(Filename.s, Seed=0, *ProgressCallback=0, UserData=0)
    Protected Result.l, File.i, FSize.q, BytesRead.i
    Protected *Buffer, State.XXH32_State
    Protected *P.ProgressCallback, Block.i, Time.i, PrevTime.i, Elapsed.u
    File = ReadFile(#PB_Any, Filename, #PB_File_SharedRead)
    If File
      FSize = Lof(File)
      *Buffer = AllocateMemory(#BufferSize, #PB_Memory_NoClear)
      If *Buffer
        *P = *ProgressCallback
        XXH32_Init(@State, Seed)
        BytesRead = ReadData(File, *Buffer, #BufferSize)
        While BytesRead = #BufferSize
          XXH32_Update(@State, *Buffer, #BufferSize)
          BytesRead = ReadData(File, *Buffer, #BufferSize)
          If *P
            Block + 1
            If Block & $f = 0
              Time = ElapsedMilliseconds()
              Elapsed = Time - PrevTime
              If Elapsed > 500
                *P(State\total_len, FSize, UserData)
                PrevTime = Time
              EndIf
            EndIf
          EndIf
        Wend
        CloseFile(File)
        Result = XXH32_Update(@State, *Buffer, BytesRead, #True)
        FreeMemory(*Buffer)
        If *P
          *P(State\total_len, FSize, UserData)
        EndIf
      EndIf       
      ProcedureReturn Result
    EndIf 
    ProcedureReturn 0
  EndProcedure 
 
EndModule


Example 1:
Code:
s.s = "The quick brown fox jumps over the lazy dog."
size = Len(s)
*mem = AllocateMemory(size)
PokeS(*mem, s, -1, #PB_Ascii)

UseModule XXH32
Debug XXH32_Hex(XXH32(*mem, size))


Example 2:
Code:
UseModule XXH32

Enumeration #PB_Event_FirstCustomValue
  #EventProgress
  #EventFinished
EndEnumeration

Global Filename.s, Busy

Procedure ProgressCallback(BytesRead.q, FileSize.q, UserData)
  PostEvent(#EventProgress, 0, 0, 0, 100*BytesRead/FileSize)
EndProcedure

Procedure ComputeXXH32(*Value)
  PostEvent(#EventFinished, 0, 0, 0, XXH32_FromFile(Filename, 0, @ProgressCallback()))
EndProcedure

OpenWindow(0, 0, 0, 320, 130, "XXH32", #PB_Window_SystemMenu | #PB_Window_ScreenCentered)
ButtonGadget(0, 10, 10, 300, 30, "Select file")
StringGadget(1, 10, 50, 300, 30, "", #PB_String_ReadOnly)
ProgressBarGadget(2, 10, 90, 300, 30, 0, 100, #PB_ProgressBar_Smooth)

Repeat
  Event = WaitWindowEvent()
  Select Event
    Case #PB_Event_Gadget
      If EventGadget() = 0 And Not Busy
        Busy = #True
        Filename = OpenFileRequester("Select file", "", "*.*", 0)
        SetGadgetText(1, GetFilePart(Filename))
        CreateThread(@ComputeXXH32(), 0)
      EndIf
    Case #EventProgress
      SetGadgetState(2, EventData())
    Case #EventFinished
      SetGadgetText(1, "[" + XXH32_Hex(EventData()) + "] " + GetGadgetText(1))
      Busy = #False
  EndSelect
Until Event = #PB_Event_CloseWindow

_________________
macOS 10.15 Catalina, PB 5.71 x64


Last edited by wilbert on Wed May 23, 2018 9:53 am, edited 9 times in total.

Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Mon May 21, 2018 6:10 pm 
Offline
Enthusiast
Enthusiast
User avatar

Joined: Thu Sep 17, 2015 3:39 pm
Posts: 137
:shock:

great.!!!

_________________
interested in Cybersecurity..


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Mon May 21, 2018 6:20 pm 
Offline
Addict
Addict

Joined: Sat Mar 02, 2013 9:17 am
Posts: 931
:shock:

great.!!!

_________________
http://www.nachtoptik.de


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 7:41 am 
Offline
User
User

Joined: Wed Apr 18, 2018 8:24 am
Posts: 98
Thank you, wilbert!

If I slightly modify my code from here:
viewtopic.php?p=522384#p522384

into:
Code:
Procedure.s GetHashFromFilePart(file.s, size.q=$100000)
  Protected.i hFile
  Protected.s hash
  Protected *buffer

  hFile = ReadFile(#PB_Any, file)
  If hFile
    *buffer = AllocateMemory(size)
    If *buffer
      If ReadData(hFile, *buffer, size)
        hash = Hex(XXH32::XXH32(*buffer, size), #PB_Long)
      EndIf
      FreeMemory(*buffer)
    EndIf
    CloseFile(hFile)
  EndIf

  ProcedureReturn PeekS(@hash)
EndProcedure

Debug GetHashFromFilePart("R:\1024KB.rawbytes")


I get this error message:
Quote:
---------------------------
PureBasic - Assembler error
---------------------------
PureBasic.asm [1500]:
mov esi, [p_mem]

error: undefined symbol 'p_mem'.


Your example for just a string works fine though


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 7:53 am 
Offline
PureBasic Expert
PureBasic Expert

Joined: Sun Aug 08, 2004 5:21 am
Posts: 3479
Location: Netherlands
forumuser wrote:
I get this error message:

I updated the code.
Could you try again with the modified module ?

_________________
macOS 10.15 Catalina, PB 5.71 x64


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 8:04 am 
Offline
Addict
Addict
User avatar

Joined: Fri Sep 21, 2007 5:52 am
Posts: 3402
Location: New Zealand
forumuser wrote:
Thank you, wilbert!

If I slightly modify my code from here:
viewtopic.php?p=522384#p522384

into:
Code:
Procedure.s GetHashFromFilePart(file.s, size.q=$100000)
  Protected.i hFile
  Protected.s hash
  Protected *buffer

  hFile = ReadFile(#PB_Any, file)
  If hFile
    *buffer = AllocateMemory(size)
    If *buffer
      If ReadData(hFile, *buffer, size)
        hash = Hex(XXH32::XXH32(*buffer, size), #PB_Long)
      EndIf
      FreeMemory(*buffer)
    EndIf
    CloseFile(hFile)
  EndIf

  ProcedureReturn PeekS(@hash)
EndProcedure

Debug GetHashFromFilePart("R:\1024KB.rawbytes")


I get this error message:
Quote:
---------------------------
PureBasic - Assembler error
---------------------------
PureBasic.asm [1500]:
mov esi, [p_mem]

error: undefined symbol 'p_mem'.


Your example for just a string works fine though


change line 37 in the module to

Code:
  mov rsi, [p.p_Buffer]


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 8:17 am 
Offline
User
User

Joined: Wed Apr 18, 2018 8:24 am
Posts: 98
Thanks for the fast response :)

The updated code works fine now...

Is there any way to use XXH32 on a full file, too?
Loading a 8+ GB .iso into memory would probably fail on most computers :mrgreen:


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 8:41 am 
Offline
PureBasic Expert
PureBasic Expert

Joined: Sun Aug 08, 2004 5:21 am
Posts: 3479
Location: Netherlands
forumuser wrote:
Is there any way to use XXH32 on a full file, too?
Loading a 8+ GB .iso into memory would probably fail on most computers :mrgreen:

I'll work on it but it may take a few days.

_________________
macOS 10.15 Catalina, PB 5.71 x64


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 8:48 am 
Offline
Addict
Addict

Joined: Sat Mar 02, 2013 9:17 am
Posts: 931
Alterable, simplest, it works so :

You read the file blockwise and create immediately new hashes from each block

Code:
; Pseudocode
your_hash$=Fingerprint(*address_of_the_destination_memory_buffer, buffer_size)
        your_hash$+hash_temp$
        your_hash$=Fingerprint(@your_hash$, StringByteLength(your_hash$))
        hash_temp$=your_hash$

The result from the hash is a Long, so it is also available for making this solution without strings
We can put also both results in only one Quad, this Quad replace then the hash_temp$ string
But i think, you see not a speed difference
A good blocksize for mechanical HDD is 16384

_________________
http://www.nachtoptik.de


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 9:50 am 
Offline
User
User

Joined: Wed Apr 18, 2018 8:24 am
Posts: 98
@wilbert
No problem, take your time and again, thanks a lot for providing XXH32!

@walbus
I tried to do it like this:
Code:
; Read a part of a file into memory and calculate the xxHash checksum over all parts
; Default size = 1 MB
Procedure.s GetXXHashFromFile(file.s, size.q=$100000)
  Protected.i hFile, bytesRead
  Protected.q fSize
  Protected.s hash, rollingHash
  Protected *buffer

  fSize = FileSize(file)
  hFile = ReadFile(#PB_Any, file)
  If hFile
      *buffer = AllocateMemory(size)
      If *buffer
        While fSize > 0
          bytesRead = ReadData(hFile, *buffer, size)
          hash = Hex(XXH32::XXH32(*buffer, bytesRead), #PB_Long)
          rollingHash + hash
          rollingHash = Hex(XXH32::XXH32(@rollingHash, StringByteLength(rollingHash)), #PB_Long)
          fSize - bytesRead
        Wend
        FreeMemory(*buffer)
      EndIf
    CloseFile(hFile)
  EndIf

  ProcedureReturn PeekS(@rollingHash)
EndProcedure

Debug GetXXHashFromFile("R:\1.1GB.iso")


So far it seems fine (if I got the logic right). Didn't test it with different memory blocks (I'm performing
tests normally from a ram disk / ssd). The ~ 1.1 GB large .iso file takes about 840ms to process on my
system. Maybe you can show how you would modify the function (two longs = one quad, etc.) to speed
it up (if possible)?


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 12:50 pm 
Offline
Addict
Addict

Joined: Sat Mar 02, 2013 9:17 am
Posts: 931
Try this

The best Way is, you add the resultzed Hash simple to the Filename
So you can simple see and also simplest check automatically the integrity from a file
As sample so : "MyVideo[HASH_ADEA3E2F].mp4"

Make the buffer size ever divisible by 16 and not smaller 4096

The resulted hash is other a the hash from the string version, but, this is OK :wink:

Have fun

Code:
; Read a part of a file into memory and calculate the xxHash checksum over all parts
; Default size = 1 MB
Procedure.s GetXXHashFromFile(file.s, size=32768)
  Protected.i hFile, bytesRead
  Protected.q fSize
  Protected *buffer
  Dim hash.l(1)
 
  fSize = FileSize(file)
  hFile = ReadFile(#PB_Any, file)
  If hFile
    *buffer = AllocateMemory(size)
    If *buffer
      While fSize > 0
        bytesRead = ReadData(hFile, *buffer, size)
        hash(1)=XXH32::XXH32(*buffer, bytesRead)
        hash(0)=XXH32::XXH32(@hash(0), 8)
        fSize - bytesRead
      Wend
      FreeMemory(*buffer)
    EndIf
    CloseFile(hFile)
  EndIf

  ProcedureReturn Hex(XXH32::XXH32(@hash(0), 8), #PB_Long) ; Finalize
EndProcedure

Debug GetXXHashFromFile("C:\Users\walbus\Desktop\test.mp4")

_________________
http://www.nachtoptik.de


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 1:29 pm 
Offline
PureBasic Expert
PureBasic Expert

Joined: Sun Aug 08, 2004 5:21 am
Posts: 3479
Location: Netherlands
While this is generating some kind of hash, it's not the same as the XXH32 hash for the entire file.
This requires preserving the internal state between the adding of more data which is exactly what I'm working on.
I already have a working version for myself but I need to test it more to make sure it generates the right hashes.

_________________
macOS 10.15 Catalina, PB 5.71 x64


Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 1:38 pm 
Offline
Addict
Addict

Joined: Sat Mar 02, 2013 9:17 am
Posts: 931
Yep, but, i think, for his videos and files a propietary solution is allways OK
The complexity and the collision resistance is the same
Also a speed difference should hardly be measurable with normal filecopy

_________________
http://www.nachtoptik.de


Last edited by walbus on Tue May 22, 2018 2:17 pm, edited 1 time in total.

Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 2:14 pm 
Offline
PureBasic Expert
PureBasic Expert

Joined: Sun Aug 08, 2004 5:21 am
Posts: 3479
Location: Netherlands
Code moved to first post

_________________
macOS 10.15 Catalina, PB 5.71 x64


Last edited by wilbert on Wed May 23, 2018 6:32 am, edited 1 time in total.

Top
 Profile  
Reply with quote  
 Post subject: Re: XXH32 Module (xxHash)
PostPosted: Tue May 22, 2018 4:32 pm 
Offline
User
User

Joined: Wed Apr 18, 2018 8:24 am
Posts: 98
@walbus
Thanks for your code! No relevant speed differences, though :oops:
1000 iterations, sometimes a few milliseconds faster, sometimes a few slower...


@wilbert

I've compiled https://github.com/Cyan4973/xxHash/releases/tag/v0.6.5
on Windows to make sure that I have something to compare checksums to...

Code:
D:\>xxhsum.exe -H0 "R:\1024KB.rawbytes"
25ad5dd3  R:\1024KB.rawbytes


-H0 uses the 32bit hash

Code:
Debug Hex(XXH32::XXH32_FromFile("R:\1024KB.rawbytes"), #PB_Long)

[17:26:29] 25AD5DD3


Checksums are fine.

Now I take a different file and do the same...

Code:
D:\>xxhsum.exe -H0 "R:\1.1GB.iso"
b6fe8a29  R:\1.1GB.iso


Code:
Debug Hex(XXH32::XXH32_FromFile("R:\1.1GB.iso"), #PB_Long)

[17:27:15] FFFFFFFFB6FE8A29


That checksum contains the correct one (b6fe8a29) but it is affixed with "FFFFFFFF".
The function returns a long and the Hex() takes a long here as well. Should I just
strip it or is there a more elegant way?

Btw, your implementation is very fast, ~750ms to hash the 1.1 GB file!
This is nearly exactly the time the C variant from above needs...


Top
 Profile  
Reply with quote  
Display posts from previous:  Sort by  
Post new topic Reply to topic  [ 19 posts ]  Go to page 1, 2  Next

All times are UTC + 1 hour


Who is online

Users browsing this forum: No registered users and 4 guests


You cannot post new topics in this forum
You cannot reply to topics in this forum
You cannot edit your posts in this forum
You cannot delete your posts in this forum

Search for:
Jump to:  

 


Powered by phpBB © 2008 phpBB Group
subSilver+ theme by Canver Software, sponsor Sanal Modifiye