PureBasic Forums - English

Code: Select all

Global.i FORMAT_BYTE = StringByteLength("a", #PB_Ascii)
CompilerIf #PB_Compiler_Unicode
  FORMAT_BYTE = StringByteLength("a", #PB_Unicode)
CompilerEndIf

Code: Select all

;-Tokenizer
Procedure.s GetCurrentChar()
  Protected car.s
  
  ;car = Chr(PeekC(@LINE+CurrentPos)) ;ORIGINAL
  car = Chr(PeekA(@LINE+CurrentPos)) ;VERSION 1
  ;car = PeekS(@LINE+CurrentPos,1,#PB_Ascii) ;VERSION 2
  ProcedureReturn car
EndProcedure

Procedure.i SkipSpace()
  Protected nbspace.i = 0
  
  While GetCurrentChar() = " " Or GetCurrentChar() = #TAB$
    CurrentPos + FORMAT_BYTE
    nbspace + 1
  Wend
  
  ProcedureReturn nbspace  
EndProcedure

Procedure.s GetToken()
  Protected sTok.s = "", c.s
  Repeat
    c = GetCurrentChar()
    ; Car = '"' and Not inside a string ?     //String definition
    If c = #DBL_QUOTE And _QUOTE = #False
      sTok+c
      CurrentPos + FORMAT_BYTE
      _QUOTE = #True
      c = GetCurrentChar()
    EndIf
    ; Car = '"' and inside a string ?         //String definition
    If c = #DBL_QUOTE And _QUOTE = #True
      _QUOTE = #False
    EndIf
    ; Car = TAB and Not inside a string ?     //Tabulation
    If c = #CHAR_TAB And _QUOTE = #False
      CurrentPos + FORMAT_BYTE
      Break
    EndIf
    ; Car = ';' or Car = '*' in the beginning 
    ; of the line and Not inside a string ?   //Remark
    If c = ";" And _QUOTE = #False
      CurrentPos = LenLine
      Break
    EndIf
    If c = "*" And _QUOTE = #False And CurrentPos = 0
      CurrentPos = LenLine : sTok = ""
    EndIf
    ; Current car position >= current Line length ?
    If CurrentPos >= LenLine
      Break
    EndIf
    ; if it's a space outside a quoted string
    If c = " " And _QUOTE = #False
      CurrentPos + FORMAT_BYTE
      Break
    EndIf
    ; Make the Token
    CurrentPos + FORMAT_BYTE
    sTok + c
  ForEver  
  ProcedureReturn sTok
EndProcedure

Code: Select all

Global.s LINE = "   BAS2H	EQU $2B" ; TAB inside
CurrentPos = 0:PosTok = 1

;*** IMPORTANT TO MULTIPLY HERE ***
LenLine = Len(LINE)*FORMAT_BYTE           ;For ASCII/UNICODE

Debug "Line to tokenize: "+LINE+" - (len:"+Str(LenLine)+")"

While CurrentPos < LenLine
  nbspace = SkipSpace()
  If nbspace > 0                          ;going to the next token
    PosTok + 1
  EndIf
  a$=GetToken()
  If a$ <> ""
    Debug a$+" ["+Str(PosTok)+"]"
    PosTok + 1
  EndIf
  SkipSpace()
Wend

Code: Select all

#CharSize = SizeOf(Character)

Structure CharStructure
  StructureUnion
    c.c
    s.s{1}
  EndStructureUnion
EndStructure

S.s = "String"

*CharPtr.CharStructure = @S

While *CharPtr\c
  
  Debug *CharPtr\s + " value : " + *CharPtr\c
  
  *CharPtr + #CharSize
  
Wend

Code: Select all

Structure CharStructure
  StructureUnion
    c.c
    s.s{1}
  EndStructureUnion
EndStructure

Structure CharArray
  p.CharStructure[0]
EndStructure

S.s = "String"

*CharArray.CharArray = @S

CurrentPos = 0

While *CharArray\p[CurrentPos]\c
  
  Debug *CharArray\p[CurrentPos]\s + " value : " + *CharArray\p[CurrentPos]\c
  
  CurrentPos + 1
  
Wend