Page 1 of 1
InsertJSONList() Crashed on bigger Lists
Posted: Sat Apr 06, 2024 4:54 am
by Bisonte
Windows 10 x64 - PB 6.xx x86 !
InsertJSONList() crashed if the list gets bigger.
In my following example, an IMA occurs directly at Max = 200,000 .... at 100,000 not yet.
It doesn't matter whether you use ASM or C-Backend.
With x64 the whole thing is no problem,
it rushes through without an error message, no matter how large the list is.
Code: Select all
Structure s_resultItem
AsString.s
AsNumber.q
EndStructure
Structure s_result
Map Column.s_resultItem()
EndStructure
NewList Row.s_result()
Procedure.i ExportResults(FileName.s, List Row.s_result())
Protected jSon = CreateJSON(#PB_Any, #PB_JSON_NoCase)
Protected Result = #False
If jSon
Debug ListSize(Row())
InsertJSONList(JSONValue(jSon), Row())
Result = SaveJSON(jSon, FileName, #PB_JSON_PrettyPrint)
FreeJSON(jSon)
EndIf
ProcedureReturn Result
EndProcedure
Filename.s = "R:\testfile.json" ; Change it !
Max = 200000 ; IMA at InsertJSONList()
;Max = 100000 ; No Error
For i=1 To Max
AddElement(Row())
For x = 65 To 89
Row()\Column(Chr(x))\AsString = Chr(x) + "a1"
Row()\Column(Chr(x))\AsNumber = Random(200000, 0)
Next x
Next i
ExportResults(Filename, Row())
Re: InsertJSONList() Crashed on bigger Lists
Posted: Sat Apr 06, 2024 9:35 am
by juergenkulow
Code: Select all
; InsertJSONList with 19,000,000 list elements Win x86 ERROR_NOT_ENOUGH_MEMORY
NewList Names.s()
For i=1 To 19000000
AddElement(Names()): Names() = Str(i)
Next
If CreateJSON(0)
InsertJSONList(JSONValue(0), Names())
Debug "Ready"
EndIf
; [09:54:13] Warte auf den Start des Executable...
; [09:54:13] Executable-Typ: Windows - x86 (32bit, Unicode)
; [09:54:13] Executable gestartet.
; [09:54:25] [ERROR] Zeile: 7
; [09:54:25] [ERROR] Ungültiger Speicherzugriff. (Schreibfehler an der Adresse 3)
; 00424EAC | 8848 03 | mov byte ptr ds:[eax+3],cl |
; EAX : 00000000
; EBX : 7EFDE000
; ECX : 77B56300 &L"10597318"
; EDX : 00000016
; EBP : 0018FF2C
; ESP : 0018FF18
; ESI : 4B7DFFB8
; EDI : 003808D8 &"Ô\t8"
; EIP : 00424EAC insertjsonlist.00424EAC
; EFLAGS : 00010246
; ZF : 1 "«««««îþ"
; OF : 0 "««îþîþîþ"
; CF : 0
; PF : 1
; SF : 0
; TF : 0 L'Ā'
; AF : 0 L'Ā'
; DF : 0 "îþ««««««««îþîþîþ"
; If : 1
; LastError : 00000008 (ERROR_NOT_ENOUGH_MEMORY)
; LastStatus : C0000017 (STATUS_NO_MEMORY)
; 0018FF18 2B261984
; 0018FF1C 00424BE1 zurück zu insertjsonlist.00424BE1 von insertjsonlist.00424E8A
; 0018FF20 4B7DFFB8
; 0018FF24 0000000A
; 0018FF28 00000000
; 0018FF2C 4F448684
; 0018FF30 004010FA zurück zu insertjsonlist.004010FA von insertjsonlist.00424BC1
; 0018FF34 4B7DFFB8
; 0018FF38 003808D8 &"Ô\t8"
; 0018FF3C 00000000
; 0018FF40 004021FB zurück zu insertjsonlist.004021FB von insertjsonlist.00401000
; 0018FF44 00400000 insertjsonlist.00400000
; 0018FF48 00000000
; 0018FF4C 008C625F
; 0018FF50 0000000A
; 0018FF54 9C4F5EE8
; 0018FF58 00000000
; 0018FF5C 00000000
; 0018FF60 7EFDE000
; 0018FF64 D7C46F00
; 0018FF68 00000066
; 0018FF6C 0051BFE7
; 0018FF70 0018FF54 "è^Oœ"
; 0018FF74 4B8B31E9 &L"12731659"
; 0018FF78 0018FFC4 Zeiger auf SEH_Record [1]
; 0018FF7C 004181F0 insertjsonlist.004181F0
; 0018FF80 9C136688
; 0018FF84 00000000
; 0018FF88 0018FF94
; 0018FF8C 75F7337A zurück zu kernel32.75F7337A von ???
; or
; 0043CC75 | 66:890A | mov word ptr ds:[edx],cx |
; EAX : 00000000
; EBX : 00000000
; ECX : 00000031 '1'
; EDX : 00000000
; EBP : 0018FEE4
; ESP : 0018FEE0
; ESI : 2C0EC332 L"0668477"
; EDI : 2C0EC330 L"10668477"
; EIP : 0043CC75 insertjsonlist.0043CC75
; EFLAGS : 00010246
; ZF : 1
; OF : 0
; CF : 0 "««««««îþ"
; PF : 1
; SF : 0
; TF : 0 L'Ā'
; AF : 0 L'Ā'
; DF : 0 "îþ««««««««îþîþîþ"
; If : 1
; LastError : 00000008 (ERROR_NOT_ENOUGH_MEMORY)
; LastStatus : C0000017 (STATUS_NO_MEMORY)
; 0018FEE0 77B1FD2C
; 0018FEE4 0018FF0C
; 0018FEE8 00424F21 zurück zu insertjsonlist.00424F21 von insertjsonlist.0043CC61
; 0018FEEC 00000000
; 0018FEF0 2C0EC330 L"10668477"
; 0018FEF4 7EFDE000
; 0018FEF8 003D08D8 &"Ô\t="
; 0018FEFC 2B150E44
; 0018FF00 00424B25 zurück zu insertjsonlist.00424B25 von insertjsonlist.00424ED9
; 0018FF04 77B1FD2C
; 0018FF08 2C0EC330 L"10668477"
; 0018FF0C 0018FF2C
; 0018FF10 00424C14 zurück zu insertjsonlist.00424C14 von insertjsonlist.00424AC8
; 0018FF14 77B1FD2C
; 0018FF18 00000008
; 0018FF1C 0000000C
; 0018FF20 2B150E4C &L"10668477"
; 0018FF24 0000000A
; 0018FF28 00000000
; 0018FF2C 4F338684
; 0018FF30 004010FA zurück zu insertjsonlist.004010FA von insertjsonlist.00424BC1
; 0018FF34 4B6CFFB8
; 0018FF38 003D08D8 &"Ô\t="
; 0018FF3C 00000000
; 0018FF40 004021FB zurück zu insertjsonlist.004021FB von insertjsonlist.00401000
; 0018FF44 00400000 insertjsonlist.00400000
; 0018FF48 00000000
; 0018FF4C 0056625F
; 0018FF50 0000000A
; 0018FF54 89E32058
; 0018FF58 00000000
; 0018FF5C 00000000
; 0018FF60 7EFDE000
; 0018FF64 7FDAAC00
; 0018FF68 00000066
; 0018FF6C 00E3091F
; 0018FF70 0018FF54 "X ã‰"
; 0018FF74 F6398C57
; 0018FF78 0018FFC4 Zeiger auf SEH_Record [1]
; 0018FF7C 004181F0 insertjsonlist.004181F0
; 0018FF80 89BF1838
; 0018FF84 00000000
; 0018FF88 0018FF94
; 0018FF8C 75F7337A zurück zu kernel32.75F7337A von ???
Re: InsertJSONList() Crashed on bigger Lists
Posted: Sat Apr 06, 2024 10:44 am
by infratec
It is definately not a problem of 'not enough memory'
The problem occours only if the map have a structure.
Code: Select all
EnableExplicit
Structure s_resultItem
AsNumber.i
EndStructure
Structure s_result
Map Column.i();s_resultItem()
EndStructure
Procedure ExportResults(List Row.s_result())
Protected.i jSon
jSon = CreateJSON(#PB_Any)
If jSon
Debug ListSize(Row())
Debug JSONValue(jSon)
InsertJSONList(JSONValue(jSon), Row())
FreeJSON(jSon)
EndIf
EndProcedure
#Max = 200000 ; IMA at InsertJSONList()
;Max = 100000 ; No Error
Define i.i, x.i
NewList Row.s_result()
For i=1 To #Max
AddElement(Row())
For x = 65 To 90
;Row()\Column(Chr(x))\AsNumber = Random(200000)
Row()\Column(Chr(x)) = Random(200000)
Next x
Next i
ExportResults(Row())
If you use the structure -> IMA
Re: InsertJSONList() Crashed on bigger Lists
Posted: Sat Apr 06, 2024 10:53 am
by Bisonte
Thanks for the workaround, that will help to export databasetables.
But is this a bug in the x86 Version ?
Because it is in all PB6 x86 versions .... ( I dont tested the versions <6)
Edit :
The Workaround is not working with datas. I tried to make a new list with only a map column.s() and it's the same...
I have at least 220k rows with datas in 18 fields. and in both cases (Map with structure and only map as string)
it crashed....
Re: InsertJSONList() Crashed on bigger Lists
Posted: Wed Apr 10, 2024 2:24 pm
by Fred
Your code needs 8GB or ram to run here on x64, so it crashs on x86 when hitting the 2GB limits. You can see this in the process monitor. This is due to the fact than the internal representation of a JSON is a tree with a lot of pointers for key/values, dynamically allocated maps for the elements handling so it's much bigger than the resulting file (which is already 500MB).
Re: InsertJSONList() Crashed on bigger Lists
Posted: Wed Apr 10, 2024 4:05 pm
by tored
If you are working with large data sets it is better to write one JSON row at a time
Code: Select all
EnableExplicit
DisableDebugger
#MAX = 200000
#FILEPATH = "testfile.json" ; Change it !
Structure s_resultItem
AsString.s
AsNumber.q
EndStructure
NewMap columns.s_resultItem()
Define file = CreateFile(#PB_Any, #FILEPATH, #PB_UTF8)
If Not file
MessageRequester("Error", "Failed creating file " + #FILEPATH, #PB_MessageRequester_Error)
End 1
EndIf
#JSON_ID = 0
If Not CreateJSON(#JSON_ID, #PB_JSON_NoCase)
MessageRequester("Error", "Failed creating JSON", #PB_MessageRequester_Error)
End 1
EndIf
WriteStringN(file, "[")
Define i, x, size, *buffer, bufferSize
For i = 1 To #MAX
For x = 65 To 89
columns(Chr(x))\AsString = Chr(x) + "a1"
columns(Chr(x))\AsNumber = Random(200000, 0)
Next x
InsertJSONMap(JSONValue(#JSON_ID), columns())
size = ExportJSONSize(#JSON_ID)
If size > bufferSize
*buffer = ReAllocateMemory(*buffer, size, #PB_Memory_NoClear)
If Not *buffer
MessageRequester("Error", "Failed allocating memory size " + size, #PB_MessageRequester_Error)
End 1
EndIf
bufferSize = size
EndIf
If Not ExportJSON(#JSON_ID, *buffer, size)
MessageRequester("Error", "Failed exporting JSON to buffer", #PB_MessageRequester_Error)
End 1
EndIf
If Not WriteData(file, *buffer, size)
MessageRequester("Error", "Failed writing JSON buffer to file", #PB_MessageRequester_Error)
End 1
EndIf
If i = #MAX
WriteStringN(file, "")
Else
WriteStringN(file, ",")
EndIf
ClearMap(columns())
ClearJSONMembers(JSONValue(#JSON_ID))
Next i
FreeJSON(#JSON_ID)
FreeMemory(*buffer)
WriteStringN(file, "]")
CloseFile(file)
; Just to verify the JSON file if small enough
;
; If Not CreateJSON(#JSON_ID)
; MessageRequester("Error", "Failed creating JSON", #PB_MessageRequester_Error)
; End 1
; EndIf
; If Not LoadJSON(#JSON_ID, #FILEPATH, #PB_JSON_NoCase)
; MessageRequester("Error", "Failed loading JSON from file " + #FILEPATH, #PB_MessageRequester_Error)
; End 1
; EndIf
; If IsJSON(#JSON_ID)
; MessageRequester("Success", "File has valid JSON", #PB_MessageRequester_Info)
; Else
; MessageRequester("Error", "File has invalid JSON", #PB_MessageRequester_Error)
; EndIf
; FreeJSON(#JSON_ID)
Small optimization, no need to reallocate buffer if JSON row already fits in buffer, now checks bufferSize