PureBasic Forums - English

Code: Select all

EnableExplicit

UseBriefLZPacker()
UseZipPacker()
UseLZMAPacker()


Define t1 = ElapsedMilliseconds()
Define url.s = "https://raw.githubusercontent.com/json-iterator/test-data/refs/heads/master/large-file.json"
Define NewMap headers.s()
headers("Accept-Encoding") = "gzip"
Debug url
Define req = HTTPRequestMemory(#PB_HTTP_Get, url, 0, 0, 0, headers())
Debug "Timer: " + Str(ElapsedMilliseconds() - t1)


If req
  Define stat.s = HTTPInfo(req, #PB_HTTP_StatusCode)
  Define res.s = HTTPInfo(req, #PB_HTTP_Response)
  Debug Left(res, 100) + "..."
  Define *buf = HTTPMemory(req)
  
;   CreateFile(0, "C:\temp\json.gzip")
;   WriteData(0, *buf, MemorySize(*buf))
;   CloseFile(0)


  Define res2.s = PeekS(*buf, MemorySize(*buf), #PB_UTF8 | #PB_ByteLength)
  Debug Left(res2, 100) + "..."
  Define *buf2 = AllocateMemory(MemorySize(*buf) * 4)

  ;ShowMemoryViewer(*buf, MemorySize(*buf))
  Debug UncompressMemory(*buf, MemorySize(*buf), *buf2, MemorySize(*buf2), #PB_PackerPlugin_Zip)
  res2.s = PeekS(*buf, MemorySize(*buf), #PB_UTF8 | #PB_ByteLength)
  Debug Left(res2, 100) + "..."
  
  FreeMemory(*buf)
  FreeMemory(*buf2)
  
  FinishHTTP(req)
EndIf

Code: Select all

EnableExplicit

#LibCurl_ExternalDLL = #True
XIncludeFile "libcurl.pbi"


Define.i curl, res
Define UserData.libcurl_userdata_structure

curl_global_init(#CURL_GLOBAL_DEFAULT)

curl = curl_easy_init();
If curl
  ;curl_easy_setopt_str(curl, #CURLOPT_URL, "https://raw.githubusercontent.com/json-iterator/test-data/refs/heads/master/large-file.json")
  ;curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
  ;curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
  
  curl_easy_setopt_str(curl, #CURLOPT_URL, "http://httpbin.org/gzip")
  
  curl_easy_setopt_str(curl, #CURLOPT_ACCEPT_ENCODING, "gzip")
  
  curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @LibCurl_WriteFunction())
  curl_easy_setopt(curl, #CURLOPT_WRITEDATA, @UserData)
  
  ; to see that the content was sent with gzip (compare Content-Length and the real size)
  curl_easy_setopt(curl, #CURLOPT_HEADER, 1)
  
  res = curl_easy_perform(curl)
  If res = #CURLE_OK
    If UserData\Memory
      Debug PeekS(UserData\Memory, MemorySize(UserData\Memory), #PB_UTF8|#PB_ByteLength)
      Debug ""
      Debug "unzipped length: " + Str(MemorySize(UserData\Memory))
    EndIf
  Else
    Debug "Error: " + curl_easy_strerror(res)
  EndIf
  
  curl_easy_cleanup(curl)
EndIf

curl_global_cleanup()

Code: Select all

EnableExplicit

#Z_BUFFER_SIZE = 1024 * 1024 ;- buffer size

#ZLIB_VERSION = "1.2.8"

#Z_OK = 0
#Z_STREAM_END = 1
#Z_FULL_FLUSH = 3
#Z_FINISH = 4

#ENABLE_GZIP = 16

Structure z_stream Align #PB_Structure_AlignC
	*next_in.BYTE
	avail_in.l
	total_in.l ;uLong
	
	*next_out.BYTE
	avail_out.l
	total_out.l ;uLong
	
	*msg.BYTE
	*state
	
	zalloc.i
	zfree.i
	opaque.i
	
	data_type.l
	adler.l ;uLong
	reserved.l ;uLong
			   ;without this, the inflateInit2() fails with a version error
	CompilerIf #PB_Compiler_Processor = #PB_Processor_x64
		alignment.l
	CompilerEndIf
EndStructure

ImportC "zlib.lib"
	inflateInit2_.l(*stream.z_stream, windowBits.l, *version, streamsize.l)
	inflate.l(*stream.z_stream, flush.l)
	inflateEnd.l(*stream.z_stream)
EndImport

Procedure ungzip(*buf)	
	Protected gzip_strm.z_stream, gzip_opaque.l, *gzip_buffer, *gzip_out, gzip_result.l, gzip_unpacked_size.l	
	Protected buf_memory_size = MemorySize(*buf)
	Protected *buf2
	
	If buf_memory_size > 0
		Debug "gzip_packed_size = " + Str(buf_memory_size)
		gzip_strm.z_stream
		gzip_strm\next_in = *buf
		gzip_strm\avail_in = buf_memory_size
		gzip_strm\opaque = @gzip_opaque
		
		inflateInit2_(gzip_strm, 15 | #ENABLE_GZIP, #ZLIB_VERSION, SizeOf(z_stream))
		
		*gzip_buffer = AllocateMemory(#Z_BUFFER_SIZE)
		*gzip_out    = AllocateMemory(#Z_BUFFER_SIZE)
		
		If *gzip_buffer And *gzip_out
			Repeat
				gzip_strm\next_out = *gzip_buffer
				gzip_strm\avail_out = #Z_BUFFER_SIZE
				gzip_result = inflate(gzip_strm, #Z_FULL_FLUSH)
				
				If gzip_result = #Z_OK Or gzip_result = #Z_STREAM_END Or gzip_strm\avail_in = 0
					CopyMemory(*gzip_buffer, *gzip_out + MemorySize(*gzip_out) - #Z_BUFFER_SIZE, #Z_BUFFER_SIZE)
					If gzip_result = #Z_STREAM_END Or gzip_strm\avail_in = 0
						Break
					Else
						*gzip_out = ReAllocateMemory(*gzip_out, MemorySize(*gzip_out) + #Z_BUFFER_SIZE)
					EndIf
				Else
					If gzip_strm\msg
						Debug PeekS(gzip_strm\msg, #PB_UTF8)
					Else
						Debug "gzip_result = " + Str(gzip_result)
					EndIf
					Break
				EndIf
			ForEver
			
			gzip_unpacked_size = gzip_strm\total_out
			If gzip_unpacked_size > 0
				Debug "gzip_unpacked_size = " + Str(gzip_unpacked_size)
				Debug "compression ratio: " + StrF(buf_memory_size * 100 / gzip_unpacked_size, 1) + "%"
				
				*buf2 = AllocateMemory(gzip_unpacked_size)
				CopyMemory(*gzip_out, *buf2, gzip_unpacked_size)
			EndIf
			
			buf_memory_size = gzip_unpacked_size
			
			FreeMemory(*gzip_out)
			FreeMemory(*gzip_buffer)
			
			inflateEnd(gzip_strm)
		EndIf
		
	Else
		Debug "buf_memory_size = 0"
	EndIf
	
	ProcedureReturn *buf2
EndProcedure

Define t1 = ElapsedMilliseconds()
Define url.s = "https://raw.githubusercontent.com/json-iterator/test-data/refs/heads/master/large-file.json"
Define NewMap headers.s()
headers("Accept-Encoding") = "gzip"
Debug url
Define req = HTTPRequestMemory(#PB_HTTP_Get, url, 0, 0, 0, headers())
Debug "HTTPRequest Timer: " + Str(ElapsedMilliseconds() - t1)
If req
	Define stat.s = HTTPInfo(req, #PB_HTTP_StatusCode)
	Define res.s = HTTPInfo(req, #PB_HTTP_Response)
	Debug Left(res, 100) + "..."
	Define *buf = HTTPMemory(req)
	FinishHTTP(req)	
	
	If *buf
		Define res2.s = PeekS(*buf, MemorySize(*buf), #PB_UTF8 | #PB_ByteLength)
		Debug Left(res2, 100) + "..."
		
		Define t2 = ElapsedMilliseconds()
		Define *buf2 = ungzip(*buf)
		Debug "UnGZIP Timer: " + Str(ElapsedMilliseconds() - t2)
		
		If *buf2
			res2.s = PeekS(*buf2, MemorySize(*buf2), #PB_UTF8 | #PB_ByteLength)
			Debug Left(res2, 100) + "..."
		
			FreeMemory(*buf2)
		EndIf
		
		FreeMemory(*buf)
	EndIf
EndIf