UnpackMemory (for backward compatibility)

Share your advanced PureBasic knowledge/code with the community.
Little John
Addict
Addict
Posts: 4773
Joined: Thu Jun 07, 2007 3:25 pm
Location: Berlin, Germany

UnpackMemory (for backward compatibility)

Post by Little John »

Hi all,

sometimes we find some old code on the forum, that contains Data sections with packed data, to which UnpackMemory() is applied. Since PB 5.00 was the last version with support for UnpackMemory(), such a code will not compile with more recent PB versions.
So I wrote this little tool, which reads the source code and unpacks these Data sections, and then we can create a slightly modified version of the old code, that uses the unpacked data. The following code has to be compiled with PB 5.00 or older, of course!

See also the documentation of UnpackMemory().

Code: Select all

; Compile this program with PB 5.00 or older!
; Newer PB versions do not support UnpackMemory() any more.

EnableExplicit

#ProgTitle$ = "UnpackMemory 1.0"
#Error$ = "Error:" + #LF$
#Terminated$ = #LF$ + "Program terminated."

Enumeration 1
   #Before
   #Data
   #After
EndEnumeration


Macro HexByte (_byte_)
   "$" + RSet(Hex(_byte_,#PB_Byte), 2, "0")
EndMacro

Macro ReadBefore()
   AddElement(before$())
   before$() = RTrim(line$)
EndMacro

Macro ReadData()
   AddElement(values$())
   values$() = Trim(line$)
   numItems = CountString(values$(), ",") + 1
   type$ = Mid(values$(), 6, 1)
   Select type$
      Case "b", "a"
         ret + numItems
      Case "c"
         ret + numItems*SizeOf(Character)
      Case "w", "u"
         ret + numItems*2
      Case "l"
         ret + numItems*4
      Case "i"
         ret + numItems*SizeOf(Integer)
      Case "q"
         ret + numItems*8
      Default
         MessageRequester(#ProgTitle$, #Error$ + "Unpacking data of type '" + type$ + "' is not supported." + #Terminated$)
         End
   EndSelect
EndMacro

Macro ReadAfter()
   AddElement(after$())
   after$() = RTrim(line$)
   If check$ = "enddatasection"
      ProcedureReturn ret
   EndIf
EndMacro


Procedure.i ReadDataSection (ifn.i, format.i, List before$(), List values$(), List after$())
   ; -- read one Data section from the source file
   ; in : ifn   : number of PB source code input file
   ;      format: format of the input file
   ; out: before$()   : lines in the Data section before the actual data
   ;      values$()   : lines in the Data section which contain the data
   ;      after$()    : lines in the Data section after the actual data
   ;      return value: number of bytes stored in the Data section
   ;                    (0 if no Data section found, -1 on error)
   Protected.i s, numItems, reading=#False, ret=0
   Protected line$, check$, type$
   
   ClearList(before$())
   ClearList(values$())
   ClearList(after$())
   
   While Not Eof(ifn)
      line$ = ReadString(ifn, format)
      check$ = line$
      s = FindString(check$, ";")
      If s
         check$ = Left(check$, s-1)
      EndIf
      check$ = LCase(Trim(check$))
      
      Select reading
         Case #False
            If check$ = "datasection"
               reading = #Before
               ReadBefore()
            EndIf
            
         Case #Before
            If Left(check$, 4) = "data"
               reading = #Data
               ReadData()
            Else
               ReadBefore()
            EndIf
            
         Case #Data
            If Left(check$, 4) = "data"
               ReadData()
            Else
               reading = #After
               ReadAfter()
            EndIf
            
         Case #After
            ReadAfter()
      EndSelect
   Wend
   
   If reading = #False
      ProcedureReturn 0        ; no Data section found
   Else
      ProcedureReturn -1       ; error
   EndIf
EndProcedure


Procedure.i UnpackDataSection (List values$(), packedBytes.i, *unpacked.Integer)
   ; -- poke one Data section into memory, and then unpack that memory area
   ; in : values$()   : lines in the Data section which contain the packed data
   ;      packedBytes : number of packed bytes stored in the Data section
   ; out: *unpacked\i : pointer to unpacked data in memory
   ;      return value: number of unpacked bytes
   ;                    (0 on error)
   Protected *packed, *ptr
   Protected.i numItems, i, ret
   Protected type$, values$
   
   *packed = AllocateMemory(packedBytes)
   If *packed = 0
      ProcedureReturn 0   ; error
   EndIf
   
   *ptr = *packed
   ForEach values$()
      type$ = Mid(values$(), 6, 1)
      values$ = LTrim(Mid(values$(), 7))
      numItems = CountString(values$, ",") + 1
      Select type$
         Case "b"
            For i = 1 To numItems
               PokeB(*ptr, Val(StringField(values$,i,",")))
               *ptr + 1
            Next
         Case "a"
            For i = 1 To numItems
               PokeA(*ptr, Val(StringField(values$,i,",")))
               *ptr + 1
            Next
         Case "c"
            For i = 1 To numItems
               PokeC(*ptr, Val(StringField(values$,i,",")))
               *ptr + SizeOf(Character)
            Next
         Case "w"
            For i = 1 To numItems
               PokeW(*ptr, Val(StringField(values$,i,",")))
               *ptr + 2
            Next
         Case "u"
            For i = 1 To numItems
               PokeU(*ptr, Val(StringField(values$,i,",")))
               *ptr + 2
            Next
         Case "l"
            For i = 1 To numItems
               PokeL(*ptr, Val(StringField(values$,i,",")))
               *ptr + 4
            Next
         Case "i"
            For i = 1 To numItems
               PokeI(*ptr, Val(StringField(values$,i,",")))
               *ptr + SizeOf(Integer)
            Next
         Case "q"
            For i = 1 To numItems
               PokeQ(*ptr, Val(StringField(values$,i,",")))
               *ptr + 8
            Next
      EndSelect
   Next
   
   *unpacked\i = AllocateMemory(50*packedBytes)   ; reserve enough memory for the unpacked data
   ret = UnpackMemory(*packed, *unpacked\i)
   FreeMemory(*packed)
   
   ProcedureReturn ret
EndProcedure


Procedure.i WriteDataSection (ofn.i, format.i, List before$(), List after$(), *unpacked.Byte, unpackedBytes.i, bytesPerLine.i)
   ; -- write one unpacked memory area in the form of a Data section to the target file
   ; in : ofn          : number of file opened for output
   ;      format       : format of the output file
   ;      before$()    : lines in the Data section before the actual data
   ;      after$()     : lines in the Data section after the actual data
   ;      *unpacked    : pointer to unpacked data in memory
   ;      unpackedBytes: number of unpacked bytes
   ;      bytesPerLine : number of bytes to be written in one Data line
   ; out: return value : 1 on success, 0 on error
   Protected.i fullLines, restBytes, line, i
   
   If unpackedBytes < 1 Or bytesPerLine < 1
      ProcedureReturn 0                      ; error
   EndIf
   
   ForEach before$()
      WriteStringN(ofn, before$(), format)
   Next
   
   fullLines = Int(unpackedBytes/bytesPerLine)
   restBytes = unpackedBytes % bytesPerLine
   
   ; write all complete lines
   For line = 1 To fullLines
      WriteString(ofn, "   Data.b ", format)
      For i = 1 To bytesPerLine-1
         WriteString(ofn, HexByte(*unpacked\b) + ",", format)
         *unpacked + 1
      Next
      WriteStringN(ofn, HexByte(*unpacked\b), format)
      *unpacked + 1
   Next
   
   ; write rest of the data, if any
   If restBytes
      WriteString(ofn, "   Data.b ", format)
      For i = 1 To restBytes-1
         WriteString(ofn, HexByte(*unpacked\b) + ",", format)
         *unpacked + 1
      Next
      WriteStringN(ofn, HexByte(*unpacked\b), format)
   EndIf
   
   ForEach after$()
      WriteStringN(ofn, after$(), format)
   Next
   
   WriteStringN(ofn, "", format)
   WriteStringN(ofn, "", format)
   
   ProcedureReturn 1                         ; success
EndProcedure


Define infile$, ext$, outfile$
Define.i ifn, ofn, format, packedBytes, unpackedBytes, numSections=0
Define unpacked.Integer
NewList values$()
NewList before$()
NewList after$()

infile$ = OpenFileRequester(#ProgTitle$ + " - Choose source code file to open", "", "All files (*.*)|*.*", 0)
If infile$ = ""
   MessageRequester(#ProgTitle$, "No input file selected." + #Terminated$)
   End
EndIf

ifn = ReadFile(#PB_Any, infile$)
If ifn = 0
   MessageRequester(#ProgTitle$, #Error$ + "Can't read file" + #LF$ + "'" + infile$ + "'." + #Terminated$)
   End
EndIf
format = ReadStringFormat(ifn)

packedBytes = ReadDataSection(ifn, format, before$(), values$(), after$())
If packedBytes = 0
   MessageRequester(#ProgTitle$, "No Data section found in file" + #LF$ + "'" + infile$ + "'." + #Terminated$)
   End
EndIf

ext$ = GetExtensionPart(infile$)
outfile$ = Left(infile$, Len(infile$)-Len(ext$)-1) + "_unpacked." + ext$
ofn = CreateFile(#PB_Any, outfile$)
If ofn = 0
   MessageRequester(#ProgTitle$, #Error$ + "Can't write to file" + #LF$ + "'" + outfile$ + "'." + #Terminated$)
   End
EndIf
WriteStringFormat(ofn, format)

;===============
;   Main loop
;===============
Repeat
   numSections + 1
   
   If packedBytes = -1
      MessageRequester(#ProgTitle$, #Error$ + "Can't read Data section #" + Str(numSections) + " in file" + #LF$ + "'" + infile$ + "'." + #Terminated$)
      End
   EndIf
   
   unpackedBytes = UnpackDataSection(values$(), packedBytes, unpacked)
   If unpackedBytes = 0
      MessageRequester(#ProgTitle$, #Error$ + "Can't unpack Data section #" + Str(numSections) + "." + #Terminated$)
      End
   EndIf
   
   If WriteDataSection(ofn, format, before$(), after$(), unpacked\i, unpackedBytes, 16) = 0
      MessageRequester(#ProgTitle$, #Error$ + "Can't write unpacked Data of section #" + Str(numSections) + "." + #Terminated$)
      End
   EndIf
   FreeMemory(unpacked\i)
   
   ; process next Data section
   packedBytes = ReadDataSection(ifn, format, before$(), values$(), after$())
Until packedBytes = 0

CloseFile(ifn)
CloseFile(ofn)
MessageRequester(#ProgTitle$, Str(numSections) + " Data section(s) successfully unpacked to file" + #LF$ + "'" + outfile$ + "'." + #LF$ + "Program finished.")
User avatar
Michael Vogel
Addict
Addict
Posts: 2797
Joined: Thu Feb 09, 2006 11:27 pm
Contact:

Re: UnpackMemory (for backward compatibility)

Post by Michael Vogel »

Hopefully, the following code could also work for newer PB versions...

Code: Select all

Procedure UnpackMemory(Source,Destination)
	UseJCALG1Packer()
	UncompressMemory(Source,0,Destination,#PB_Packer_JCALG1)
EndProcedure
Little John
Addict
Addict
Posts: 4773
Joined: Thu Jun 07, 2007 3:25 pm
Location: Berlin, Germany

Re: UnpackMemory (for backward compatibility)

Post by Little John »

Michael Vogel wrote:Hopefully, the following code could also work for newer PB versions...

Code: Select all

Procedure UnpackMemory(Source,Destination)
	UseJCALG1Packer()
	UncompressMemory(Source,0,Destination,#PB_Packer_JCALG1)
EndProcedure
This does not work with PB 5.20 LTS.

However, I tried

Code: Select all

Procedure.i UnpackMemory (Source, Destination)
   UseJCALG1Packer()
   ProcedureReturn UncompressMemory(Source, 0, Destination, 0, #PB_PackerPlugin_JCALG1)
EndProcedure
:-)

According to a small test that I made, this seems to be a valid replacement for the old built-in UnpackMemory() function.
Very helpful for backward compatibility, thank you!
So my code in the first post seems to be superfluous now.
User avatar
Michael Vogel
Addict
Addict
Posts: 2797
Joined: Thu Feb 09, 2006 11:27 pm
Contact:

Re: UnpackMemory (for backward compatibility)

Post by Michael Vogel »

Little John wrote:[...] This does not work with PB 5.20 LTS.
Seems, that it is time to install 5.2 (still using 5.11) :)
davido
Addict
Addict
Posts: 1890
Joined: Fri Nov 09, 2012 11:04 pm
Location: Uttoxeter, UK

Re: UnpackMemory (for backward compatibility)

Post by davido »

Nice work, gentlemen.
DE AA EB
User avatar
Bisonte
Addict
Addict
Posts: 1305
Joined: Tue Oct 09, 2007 2:15 am

Re: UnpackMemory (for backward compatibility)

Post by Bisonte »

But :!:
Help (F1) says wrote:This packer is only available on Windows x86 (32-bit). This packer is deprecated and no more supported.
Why it not work with a "wrap" of Pack and Unpackmemory in a dll, compile it with an older version of pb and use the dll with the 5.20 ?
PureBasic 6.21 (Windows x64) | Windows 11 Pro | AsRock B850 Steel Legend Wifi | R7 9800x3D | 64GB RAM | RTX 5080 | ThermaltakeView 270 TG ARGB | build by vannicom​​
English is not my native language... (I often use DeepL.)
Little John
Addict
Addict
Posts: 4773
Joined: Thu Jun 07, 2007 3:25 pm
Location: Berlin, Germany

Re: UnpackMemory (for backward compatibility)

Post by Little John »

Great idea, Bisonte. Thank you!
User avatar
Bisonte
Addict
Addict
Posts: 1305
Joined: Tue Oct 09, 2007 2:15 am

Re: UnpackMemory (for backward compatibility)

Post by Bisonte »

Little John wrote:Great idea, Bisonte. Thank you!
Please let me know how you do it. I tried this with no success...
PureBasic 6.21 (Windows x64) | Windows 11 Pro | AsRock B850 Steel Legend Wifi | R7 9800x3D | 64GB RAM | RTX 5080 | ThermaltakeView 270 TG ARGB | build by vannicom​​
English is not my native language... (I often use DeepL.)
Little John
Addict
Addict
Posts: 4773
Joined: Thu Jun 07, 2007 3:25 pm
Location: Berlin, Germany

Re: UnpackMemory (for backward compatibility)

Post by Little John »

Bisonte wrote:Please let me know how you do it. I tried this with no success...
Ooops ... I probably misunderstood your other message (and I didn't expext any problems).
Anyway, I'll reinstall PB 5.00 today or at the weekend and try to do as you suggested. Of course I'll tell you the results then. :-)
Little John
Addict
Addict
Posts: 4773
Joined: Thu Jun 07, 2007 3:25 pm
Location: Berlin, Germany

Re: UnpackMemory (for backward compatibility)

Post by Little John »

Bisonte, your idea works fine here. :-)
(I currently can only test on Windows XP x86.)
For the test, all source files and the DLL are in the same directory.


I compiled this source code with PB 5.00 to a DLL:

Code: Select all

ProcedureDLL PB_UnpackMemory (*source, *destination)
   ProcedureReturn UnpackMemory(*source, *destination)
EndProcedure
Then I ran this test code with PB 5.00:

Code: Select all

; ## This code runs only on PB 5.00 or older, ##
; ## because of PackMemory()/UnpackMemory().  ##

EnableExplicit

Define source$, *packed, packedBytes.i, *unpacked, unp$, *pntr.Ascii, ofn.i, i.i, line$

source$ = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed "
source$ + "do eiusmod tempor incididunt ut labore et dolore magna aliqua."

; -- pack source$
*packed = AllocateMemory(StringByteLength(source$)+8)
packedBytes = PackMemory(@Source$, *packed, StringByteLength(Source$))
If packedBytes = 0
   Debug "Error with PackMemory()"
   End
EndIf
Debug Str(StringByteLength(Source$)) + " bytes --> " + Str(packedBytes) + " bytes"

; -- unpack the packed data and compare them with source$
*unpacked = AllocateMemory(StringByteLength(source$)+SizeOf(Character))
UnpackMemory(*packed, *unpacked)
unp$ = PeekS(*unpacked)
FreeMemory(*unpacked)
If unp$ <> source$
   Debug "Error: unpacked data <> source data."
   End
EndIf

; -- write PB include file, that contains source$ and packed data
ofn = CreateFile(#PB_Any, #PB_Compiler_FilePath + "Data.pbi")
If ofn = 0
   Debug "Error: Can't create output file."
   End
EndIf
WriteStringN(ofn, "DataSection")
WriteStringN(ofn, "   Source:")
WriteStringN(ofn, "   Data.s " + #DQUOTE$ + source$ + #DQUOTE$)
WriteStringN(ofn, "")
WriteStringN(ofn, "   Packed:")
line$ = "   Data.b "
*pntr = *packed
For i = 1 To packedBytes
   If i % 10 <> 0
      line$ + "$" + RSet(Hex(*pntr\a), 2, "0") + ", "
   Else
      line$ + "$" + RSet(Hex(*pntr\a), 2, "0")
      WriteStringN(ofn, line$)
      line$ = "   Data.b "
   EndIf
   *pntr + 1
Next
WriteStringN(ofn, Left(line$, Len(line$)-2))
WriteStringN(ofn, "EndDataSection")
FreeMemory(*packed)
CloseFile(ofn)
Debug "Finished."
Then I ran this test code with PB 5.20, which uses the DLL:

Code: Select all

; ## Run this code with PB 5.10 or newer,       ##
; ## in order to obtain backward compatibility. ##

EnableExplicit

XIncludeFile #PB_Compiler_FilePath + "Data.pbi"


Procedure UnpackMemory (*source, *destination)
   Protected dll$, fn.i, ret.i=0
   
   CompilerSelect #PB_Compiler_Processor
      CompilerCase #PB_Processor_x86
         dll$ = "PureBasic_5.00_x86.dll"
      CompilerCase #PB_Processor_x64
         dll$ = "PureBasic_5.00_x64.dll"
   CompilerEndSelect

   fn = OpenLibrary(#PB_Any, #PB_Compiler_FilePath + dll$)
   If fn
      ret = CallFunction(fn, "PB_UnpackMemory", *source, *destination)
      CloseLibrary(fn)
   EndIf
   ProcedureReturn ret
EndProcedure


Define source$, *unpacked, unp$

; -- read source$
source$ = PeekS(?Source)

; -- unpack the packed data and compare them with source$
*unpacked = AllocateMemory(StringByteLength(source$)+SizeOf(Character))
UnpackMemory(?Packed, *unpacked)
unp$ = PeekS(*unpacked)
FreeMemory(*unpacked)
If unp$ <> source$
   Debug "Error: unpacked data <> source data."
Else
   Debug "OK"
EndIf
Note that the test codes have to be compiled in the same mode, i.e. both in Unicode mode, or both not in Unicode mode. I'm pretty sure that this plays a role here only because I used a string for testing, and has nothing got to do with the DLL itself.
Post Reply