FastStringSplit improve speed ...
Posted: Fri Dec 22, 2017 9:43 am
hello everyone

Code: Select all
EnableExplicit
ImportC ""
wcsstr.i(*str1, *str2)
_wcslwr.i(*cs)
wcslen.l(*cs)
EndImport
DisableDebugger
Procedure FastStringSplit(sString.s, sDelimiters.s, Array StringArray.i(1), Casesense = 1)
Protected String = @sString
Protected Delimiters = @sDelimiters
Protected StringSize = wcslen(String) * 2
Protected DelimitersSize = wcslen(Delimiters) * 2
Protected aString = AllocateMemory(StringSize + 2):CopyMemory(String,aString,StringSize)
If Not aString:ProcedureReturn 0:EndIf
If Casesense = 0
_wcslwr(String)
_wcslwr(Delimiters)
EndIf
Protected String2 = String
Protected ElementStart ,ElementSize
Protected aItem
Repeat
String2 = wcsstr(String2,Delimiters)
If String2 = 0
ElementSize = StringSize - ElementStart
Else
ElementSize = String2 - (String + ElementStart)
EndIf
If ArraySize(StringArray()) < aItem : ReDim StringArray(aItem +99):EndIf
StringArray(aItem) = aString + ElementStart
aItem + 1
PokeW(aString + ElementStart + ElementSize, 0)
ElementStart + ElementSize + DelimitersSize
String2 + DelimitersSize
Until String2 = DelimitersSize
If ElementSize = StringSize
FreeMemory(aString)
ProcedureReturn 0
EndIf
ReDim StringArray(aItem-1)
ProcedureReturn aString
EndProcedure
Procedure.i SplitStringStringField(String.s, Delimiter.s, Array Output.s(1))
Protected n = 1 + CountString(String, Delimiter),i
Dim Output.s(n - 1)
For i = 0 To n-1
Output(i) = Trim(StringField(String, 1+i, Delimiter))
Next i
EndProcedure
Procedure ExplodeStringIntoArray(Separator$, String$, Array Output$(1))
Protected ArrayIndex, StartPos, SeparatorPos
ArrayIndex = -1
StartPos = 1
Repeat
ArrayIndex + 1
If ArraySize(Output$()) < ArrayIndex
ReDim Output$(ArrayIndex + 99) ; Wenn Output-Array zu klein ist, es um 100 Elemente vergrößern
EndIf
SeparatorPos = FindString(String$, Separator$, StartPos)
If SeparatorPos = 0
Output$(ArrayIndex) = Mid(String$, StartPos)
Break
EndIf
Output$(ArrayIndex) = Mid(String$, StartPos, SeparatorPos - StartPos)
StartPos = SeparatorPos + 1
ForEver
ReDim Output$(ArrayIndex) ; Array auf die wirklich notwendige Größe setzen
ProcedureReturn ArrayIndex
EndProcedure
EnableDebugger
DisableExplicit
selectfile.s = OpenFileRequester("Select big texte...","","txt|*.*;*.txt",0)
Dim StringArray.i(1000)
Procedure.s FileRead_String(FilePath.s,StringType = #PB_UTF8)
Protected hfile = ReadFile(#PB_Any, FilePath,#PB_File_SharedRead|#PB_File_SharedWrite)
If hfile
Protected SizeFile = Lof(hfile)
Protected *pFile = AllocateMemory(SizeFile+4)
ReadData(hfile,*pFile,SizeFile)
CloseFile(hfile)
EndIf
If *pFile:Protected Strfile.s= PeekS(*pFile,-1,StringType):FreeMemory(*pFile):EndIf
ProcedureReturn Strfile
EndProcedure
FileString.s = FileRead_String(selectfile)
time = ElapsedMilliseconds()
parray = FastStringSplit(FileString,#CRLF$, StringArray())
If parray
Debug "FastStringSplit : ElapsedMilliseconds = " +
Str(ElapsedMilliseconds() - time ) + " / ArraySize = " +
Str(ArraySize(StringArray()))
; For r = 0 To ArraySize(StringArray())
; Debug PeekS(StringArray(r))
; Next
FreeMemory(parray)
Else
EndIf
Dim Word.s(0)
time = ElapsedMilliseconds()
SplitStringStringField(FileString, #CRLF$, Word())
Debug "SplitStringStringField : ElapsedMilliseconds = " +
Str(ElapsedMilliseconds() - time ) + " / ArraySize = " +
Str(ArraySize(Word()))
time = ElapsedMilliseconds()
ExplodeStringIntoArray(#CRLF$, FileString, Word())
Debug "ExplodeStringIntoArray : ElapsedMilliseconds = " +
Str(ElapsedMilliseconds() - time ) + " / ArraySize = " +
Str(ArraySize(Word()))
; tested With "1.5 mb" text file
; FastStringSplit : ElapsedMilliseconds = 14 / ArraySize = 18710
; SplitStringStringField : ElapsedMilliseconds = 60798 / ArraySize = 18710
; ExplodeStringIntoArray : ElapsedMilliseconds = 28625 / ArraySize = 18710