[Crossplatform] HTTP GET / Parallels Download
Posted: Fri Dec 26, 2008 7:06 am
This tips do parallels download.
1.CreateList
2.AddElement, and Information set
3.Call PureHTTP_Get_File(myfile())
4.Result retrun, and finish
Normal DL X 2 or 3 speed. have fun.
Library
Test Code
1.CreateList
2.AddElement, and Information set
3.Call PureHTTP_Get_File(myfile())
4.Result retrun, and finish
Normal DL X 2 or 3 speed. have fun.
Library
Code: Select all
; PureHTTP Include Parallels - Version: 0.44b + custmize 0.2
; OS: Linux / Windows / AmigaOS / MacOSX
; Author: Marius Eckardt (Thalius), oryaaaa
; a lil lib for HTTP ( for now just Downloading ;)
; Last Changes: 06.08.2008
;
; PureHTTP is aimed to be a HTTP 1.0-Compatible Crossplatform Library for use in your Projects.
; This code is PublicDomain - Credits apprechiated if you use it tho ;)
;
; Thx go to mp303 who brought me on the Idea initially with some Networkquestions in the Bugsection. :)
;
; Note: This Code is supposed to be fast but not super optimized. Basically ok for the Job - if you do optimizations
; / improvements you feel the Community could benefit from feel free to post them. :)
;
; Features:
; - HTTP Download to File with :
; - HTTP HEADER Info
; - HTTP Error Handling
; - HTTP Redirect support ( should work now! )
; - Connection Timeout Handling
; - Callback Support
; - Download Timer
; - KB/s o Meter
; - Limit Download Speed
;
; Changes:
; 0.2b:
; - Added Callback Function Support
; 0.3b:
; - lil Optimization
; 0.4b:
; - more Optimizations
; - Fixed Speedlimit To a per Kb/s Value
; - Added PureHTTP_SplitURL(url.s, type.l) - Splits an URL to Host/Path/Filename
; - Extended Example a little
; 0.41 - 0.44b:
; - small Optimizations
; - Added Redirect Support
; - Download to Memory ( Weee ! ;)
; - Added fixes by "moogle"
; 0.44b + Customize oryaaaaa
; - Parallels Downlads
; - Unicord Support
;
; ToDo:
; - Bugfixes ?
; - Extend HTTP Header Structure to support more of HTTP/1.0
; - HTTP Resume Support
; - GUI with Callback Example
; - HTTP POST Functionality
; - Manual
;- Konstants
; Network
#PureHTTP_Name = "PureHTTP" ; UserAgent Name
#PureHTTP_Defaultfile = "index.html" ; Default for index Files ( used to parse URLs )
#PureHTTP_Buffersize = 4092 ; Chunk-/Buffersize (This Value is balanced ok for average usage. There might be a speedgain on large files increasing this, but slowdown for smaller files)
#PureHTTP_Timeout = 10000 ; Network Timeout in ms
; Status
Enumeration
#PureHTTP_STATUS_CONNECTING ; Connecting
#PureHTTP_STATUS_CONNECTED ; Connected
#PureHTTP_STATUS_GOTHEADER ; Got Header info Filesize / Content-Type
#PureHTTP_STATUS_RECEIVE ; Receiving...
#PureHTTP_STATUS_REDIRECT ; HTTP Redirect
#PureHTTP_STATUS_IDLE ; Idle, Waiting for Data...
#PureHTTP_STATUS_FINISHED ; Download Finished
#PureHTTP_STATUS_TIMEOUT ; Timeout
#PureHTTP_STATUS_HTTPERROR ; HTTP Error / Return
#PureHTTP_STATUS_FILEERROR ; Can't create file
#PureHTTP_STATUS_ABORT ; Download Aborted
EndEnumeration
; HTTP URL Related
#PureHTTP_URL_Protocol = "http://"
; URL Return-Selection Konstants
Enumeration
#PureHTTP_URL_TO_HOST ; Host
#PureHTTP_URL_TO_PATH ; Path
#PureHTTP_URL_TO_FILE ; Filename
EndEnumeration
; HTTP Protocol Related
#HTTP_Port = 80
#HTTP_Protocol10 = "HTTP/1.0"
#HTTP_Protocol11 = "HTTP/1.1"
#HTTP_ContentLength = "Content-Length:"
#HTTP_ContentType = "Content-Type:"
; HTTP Retruncodes
#HTTP_CODE_OK = 200 ; OK
#HTTP_CODE_REDIRECT = 301 ; Redirect
#HTTP_CODE_TREDIRECT = 307 ; Transparent Redirect
;- Structures
; HTTP Header Struct
Structure _PureHTTP_HEADER
Returncode.l ; HTTP Return Code -> 200 = OK -> 404 = File Not Found
ContentLength.l ; Reported Content-Size
ContentType.s ; Reported Content-Type
HTTPReturnString.s ; HTTP Retrunstring -> "HTTP ERROR 404 Not Found"
EndStructure
; File Struct
Structure _PureHTTP_GET_FILE
; File
id.l ; FileIDThread ( in case we download multiple files at once )
*Callback ; Callback Function
ConID.l ; ConnectionID
Host.s ; Host to Connect to
Path.s ; Path to File on Host
outputfile.s ; Path to local Filetarget
*DestBuffer ; Pointer to Destination Memory Buffer ( Alternative to File Download )
Status.l ; Download Status
; HTTP Header Meta Info
Header._PureHTTP_HEADER
; Network
*Buffer ; Pointer to our Paketbuffer
Totalbytes.l ; Holds actual Total bytes received from this File
StartTimer.l ; Start Timer in ms
Timer.l ; Current Our Timer in ms
Totaltime.l ; Total DownloadTime
limitspeed.l ; Limit Downloadspeed in Kb/s
kbps.f ; KB/s
EndStructure
;- Procs
; -- Semi Internal Functions
; Builds GET HTTP Header, kinda interna therefor in CAPS
; Returns HTTP HeaderString
Procedure.s PureHTTP_BUILDHEADER_HTTP_GET(*this._PureHTTP_GET_FILE)
Protected extras.s
If Len(*this\Path) <= 1
*this\Path = "/"
EndIf
; Add more Checks & stuff here...
extras.s = Chr(13) + Chr(10) + "User-Agent: " + #PureHTTP_Name
ProcedureReturn "GET " + *this\Path + " " + #HTTP_Protocol10 + Chr(13) + Chr(10) + "host: "+ *this\Host + Chr(13) + Chr(10) +"Connection: Close" + extras.s + Chr(13) + Chr(10) + Chr(13) + Chr(10)
EndProcedure
; Set Download Status
; kinda Interna-helper function
; Can be used to send an event to thread
; for ex: PureHTTP_SET_STATUS(@mydownload, #PureHTTP_STATUS_ABORT )
; Aborts download.
Procedure PureHTTP_SET_STATUS(List this._PureHTTP_GET_FILE(), Status.l)
;// Set Status
this()\Status = Status.l
;// Call Callback
If this()\Callback <> 0
CallFunctionFast(this()\Callback,this())
EndIf
EndProcedure
; -- User Functions
; Func: PureHTTP_SplitURL(url.s, [ #Pure_HTTP_URL_* ])
; Desc: Little Helper Function to Extract Host/Path/Filename out of an URL
; Returns: String: Hostname, Path, Filename
; See Types:
; #PureHTTP_URL_HOST ; Host
; #PureHTTP_URL_PATH ; Path
; #PureHTTP_URL_FILE ; Filename
;
; Example: Debug PureHTTP_SplitURL("http://www.wavemage.com/edscore/5-EndTitle.mp3",#PureHTTP_URL_FILE)
; See example below for more...
Procedure.s PureHTTP_SplitURL(URL.s, type.l = #PureHTTP_URL_TO_HOST)
Protected i.l, a.l
Result.s = ""
;// Do we have a http:// ?
a.l = FindString(URL.s,#PureHTTP_URL_Protocol,0)
If a.l
URL.s = Right(URL.s,Len(URL.s) - Len(#PureHTTP_URL_Protocol))
If type.l >= #PureHTTP_URL_TO_HOST And type.l <= #PureHTTP_URL_TO_FILE
;// Host
Result.s = StringField(URL.s,0,"/")
If type.l = #PureHTTP_URL_TO_FILE
;// Filename
;// Find File
num_slash=CountString(URL.s, "/")
Result.s=StringField(URL.s, num_slash+1, "/")
;// Sure we got a File ?
If FindString(Result.s,".",0) = 0 Or Result.s = StringField(URL.s,0,"/")
;// No ? Assuming its an index HTML
Result.s = #PureHTTP_Defaultfile
EndIf
EndIf
;// Path
If type.l = #PureHTTP_URL_TO_PATH
Result.s = Right(URL.s,Len(URL.s)-Len(Result.s))
EndIf
EndIf
EndIf
ProcedureReturn Result.s
EndProcedure
; Func: PureHTTP_Get_File(@myfile)
; Returns: Statuscode See: #PureHTTP_STATUS_*
; Desc: Downloads File
Procedure PureHTTP_Get_File(List this._PureHTTP_GET_FILE())
Debug ListSize(this())
Structure HttpPacket
offset.l
eoffset.l
cloffset.l
ctoffset.l
solocoffset.l
eolocoffset.l
Bytes.l
PaketCounter.l
SEvent.l
outfile.l
m_mul.l
redirection.l
dest_buffersize.l
tmp_redir_url.s
retry.b
EndStructure
Protected NewList packet.HttpPacket()
ForEach this._PureHTTP_GET_FILE()
;// Set Status
PureHTTP_SET_STATUS(this(), #PureHTTP_STATUS_CONNECTING)
this()\ConID = OpenNetworkConnection(this()\Host , #HTTP_Port )
AddElement( packet() )
Debug this()\ConID
If this()\ConID
;// Set Status
PureHTTP_SET_STATUS(this(), #PureHTTP_STATUS_CONNECTED)
Debug "* Connected"
;// Allocate Receivebuffer
this()\Buffer = AllocateMemory(#PureHTTP_Buffersize)
;// Init Timer
this()\StartTimer = ElapsedMilliseconds()
this()\Timer = this()\StartTimer
;// Send HTTP Request
SendNetworkString(this()\ConID,PureHTTP_BUILDHEADER_HTTP_GET(this()))
Debug "* Sending Request"
;// Wait for incoming Data ...
Debug "* Waiting for Reply"
Else
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_TIMEOUT)
EndIf
Next
Repeat
FirstElement(packet())
ForEach this()
If This()\ConID
packet()\SEvent = NetworkClientEvent(this()\ConID)
Select packet()\SEvent
;// We received Data!
Case #PB_NetworkEvent_Data
Debug "* Receiving..."
;// Now go Fetch!
Repeat
packet()\Bytes.l = ReceiveNetworkData(this()\ConID, this()\Buffer, #PureHTTP_Buffersize)
;// Is this the first Paket containing the Header ?
If packet()\PaketCounter.l = 0
; PrintN(PeekS(*this\Buffer,#PureHTTP_Buffersize)) ;<- lil Debug For the header
;// Basic Process Header - Get DataOffsets
packet()\eoffset = FindString(PeekS(this()\Buffer, 200, #PB_Ascii), #HTTP_Protocol10, 0) + Len(#HTTP_Protocol10)
If packet()\eoffset = Len(#HTTP_Protocol10) ;// We got a HTTP1.1 response, find new offset.
packet()\eoffset = FindString(PeekS(this()\Buffer, 200, #PB_Ascii), #HTTP_Protocol11, 0) + Len(#HTTP_Protocol11)
EndIf
;// We sure in for a right Response now ?
If packet()\eoffset.l = Len(#HTTP_Protocol10) ; no .. still no offset
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_HTTPERROR)
Else ; Yeah baby!
packet()\cloffset.l = FindString(PeekS(this()\Buffer, #PureHTTP_Buffersize, #PB_Ascii), #HTTP_ContentLength, packet()\eoffset.l) + Len(#HTTP_ContentLength)
packet()\ctoffset.l = FindString(PeekS(this()\Buffer, #PureHTTP_Buffersize, #PB_Ascii), #HTTP_ContentType, packet()\eoffset.l) + Len(#HTTP_ContentType)
packet()\offset.l = FindString(PeekS(this()\Buffer, #PureHTTP_Buffersize, #PB_Ascii),Chr(13) + Chr(10) + Chr(13) + Chr(10),packet()\eoffset.l) + 3
;// Do we have a Content-Length Info? if so, set Size
If packet()\cloffset.l <> Len(#HTTP_ContentLength)
this()\Header\ContentLength = Val(LTrim(RTrim(PeekS(this()\Buffer+packet()\cloffset.l,FindString(PeekS(this()\Buffer+packet()\cloffset.l,#PureHTTP_Buffersize-packet()\cloffset.l, #PB_Ascii),Chr(13) + Chr(10),0), #PB_Ascii))))
EndIf
;// Set Content-Type
If packet()\ctoffset.l <> Len(#HTTP_ContentType)
this()\Header\ContentType = LTrim(RTrim(PeekS(this()\Buffer+packet()\ctoffset.l,FindString(PeekS(this()\Buffer+packet()\ctoffset.l,#PureHTTP_Buffersize-packet()\ctoffset.l, #PB_Ascii),Chr(13) + Chr(10),0), #PB_Ascii)))
EndIf
;// Select Header Response
this()\Header\Returncode = Val(PeekS(this()\Buffer+packet()\eoffset.l, 3, #PB_Ascii))
Select this()\Header\Returncode
Case #HTTP_CODE_OK
;// Create File
If Len(this()\outputfile) > 0
packet()\outfile = CreateFile(#PB_Any,this()\outputfile+".part")
EndIf
;//Set status
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_GOTHEADER)
;// Write first bytes to File
If packet()\outfile
Debug "Writing HeaderData to File..."
this()\Totalbytes = packet()\Bytes.l - packet()\offset.l
WriteData(packet()\outfile, this()\Buffer + packet()\offset.l, packet()\Bytes.l - packet()\offset.l)
Else
;// Check if Download to Memory & Set Status
If Not this()\Destbuffer
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_FILEERROR)
EndIf
EndIf
; -- Add
;// Check if Memory Reserved.
If this()\Destbuffer
packet()\dest_buffersize.l = MemorySize(this()\Destbuffer)
;// Check if Buffer is Big enough for first data...
If #PureHTTP_Buffersize > packet()\dest_buffersize.l
;// Reallocate Memory to Rec-Buffersize
this()\Destbuffer = ReAllocateMemory(this()\Destbuffer,#PureHTTP_Buffersize)
EndIf
;// Download first Data to Memory Buffer
CopyMemory(this()\Buffer + packet()\offset.l,this()\Destbuffer,packet()\Bytes.l - packet()\offset.l)
this()\Totalbytes + ( packet()\Bytes.l - packet()\offset.l)
EndIf
;// Process HTTP REDIRECT
Case #HTTP_CODE_REDIRECT To #HTTP_CODE_TREDIRECT
Debug "302 -> Redirect"
;// Set Status
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_GOTHEADER)
;// Handle Redirection - Find new Location
packet()\solocoffset.l = FindString(PeekS(this()\Buffer,0, #PB_Ascii),"Location:",packet()\eoffset.l) + 10
packet()\eolocoffset.l = FindString(PeekS(this()\Buffer,0, #PB_Ascii),Chr(13) + Chr(10),packet()\solocoffset.l)
packet()\tmp_redir_url.s = Mid(PeekS(this()\Buffer,0, #PB_Ascii), packet()\solocoffset, packet()\eolocoffset - packet()\solocoffset)
;// Got Location ? Rewrite URL !
If packet()\solocoffset.l
this()\Host = PureHTTP_SplitURL(packet()\tmp_redir_url.s,#PureHTTP_URL_TO_HOST)
this()\Path = PureHTTP_SplitURL(packet()\tmp_redir_url.s,#PureHTTP_URL_TO_PATH)
;// Set status
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_REDIRECT)
;Delay(100)
If this()\Buffer
FreeMemory(this()\Buffer)
EndIf
this()\Totaltime = ElapsedMilliseconds() - this()\StartTimer
If this()\ConID
CloseNetworkConnection(this()\ConID)
EndIf
PureHTTP_SET_STATUS(this(), #PureHTTP_STATUS_CONNECTING)
this()\ConID = OpenNetworkConnection( this()\Host , #HTTP_Port )
AddElement( packet() )
If this()\ConID
;// Set Status
PureHTTP_SET_STATUS(this(), #PureHTTP_STATUS_CONNECTED)
Debug "* Connected"
;// Allocate Receivebuffer
this()\Buffer = AllocateMemory(#PureHTTP_Buffersize)
;// Init Timer
this()\StartTimer = ElapsedMilliseconds()
this()\Timer = this()\StartTimer
;// Send HTTP Request
SendNetworkString(this()\ConID,PureHTTP_BUILDHEADER_HTTP_GET(this()))
Debug "* Sending Request"
;// Wait for incoming Data ...
Debug "* Waiting for Reply"
Else
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_TIMEOUT)
EndIf
EndIf
Default ; Unknown Error Code
;// Set Status
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_HTTPERROR)
this()\Header\HTTPReturnString = PeekS(this()\Buffer+packet()\eoffset.l,FindString(PeekS(this()\Buffer+packet()\eoffset.l,200, #PB_Ascii),Chr(13) + Chr(10),0), #PB_Ascii)
EndSelect
EndIf
Else ;// Download Finished ?
If packet()\Bytes.l = 0
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_FINISHED)
Debug "** FINISHED **"
Else ;// No? ok then write some ...
this()\Totalbytes + packet()\Bytes.l
;// Set Status
PureHTTP_SET_STATUS(this(), #PureHTTP_STATUS_RECEIVE)
;// write Data To File
If packet()\outfile
WriteData(packet()\outfile,this()\Buffer,packet()\Bytes.l)
Else
;// Set Status
If Not this()\Destbuffer
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_FILEERROR)
EndIf
EndIf
;// Download to Memory: Check if Memory Reserved.
If this()\Destbuffer
packet()\dest_buffersize.l = MemorySize(this()\Destbuffer)
;// Check if Buffer is Big enough for coming data...
If this()\Totalbytes > packet()\dest_buffersize.l
;// Reallocate Memory to Fit
packet()\m_mul.l = Round(this()\Totalbytes / #PureHTTP_Buffersize,1) + 1
this()\Destbuffer = ReAllocateMemory(this()\Destbuffer,(#PureHTTP_Buffersize * packet()\m_mul.l))
EndIf
;// Download first Data to Memory Buffer
CopyMemory(this()\Buffer,(this()\Destbuffer + this()\Totalbytes - packet()\Bytes.l),packet()\Bytes.l)
EndIf
;// Update KB/s
this()\kbps = (this()\Totalbytes / ((ElapsedMilliseconds()-this()\StartTimer)/1000)) / 1024
Debug StrF(this()\kbps) + " kb/s"
;// Update Timer
this()\Timer = ElapsedMilliseconds()
EndIf
EndIf
packet()\PaketCounter.l + 1
;// Limit Speed to
If this()\limitspeed <> 0
While this()\kbps > this()\limitspeed
;// Update KB/s & wait out our Limit...
this()\kbps = (this()\Totalbytes / ((ElapsedMilliseconds()-this()\StartTimer)/1000)) / 1024
Delay(10)
Wend
EndIf
Until this()\Status >= #PureHTTP_STATUS_FINISHED
;// Close File
If packet()\outfile
CloseFile(packet()\outfile)
CopyFile(this()\outputfile+".part",this()\outputfile)
DeleteFile(this()\outputfile+".part")
EndIf
Default
;// Set Status
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_IDLE)
Debug "- IDLE -"+Str(ListIndex(this()))
EndSelect
;// Check Timeout
If ElapsedMilliseconds() - this()\Timer > #PureHTTP_Timeout
PureHTTP_SET_STATUS(this(),#PureHTTP_STATUS_TIMEOUT)
EndIf
;// Don't burn CPU while waiting...
EndIf
NextElement(packet())
Next
Delay(10)
ForEach this()
If this()\Status >= #PureHTTP_STATUS_FINISHED
m+1
If ListSize(this())=<m
Break 2
EndIf
EndIf
Next
ForEver
;// Free Memory
ForEach this()
If this()\Buffer
FreeMemory(this()\Buffer)
EndIf
this()\Totaltime = ElapsedMilliseconds() - this()\StartTimer
If this()\ConID
CloseNetworkConnection(this()\ConID)
EndIf
; If redirection.l = 1
; PureHTTP_Get_File(*this())
; EndIf
Next
ProcedureReturn this()\Status
EndProcedure
Code: Select all
InitNetwork()
NewList myfile._PureHTTP_GET_FILE()
myurl.s = "http://www.watch.impress.co.jp/headline/rss/headline.rdf"
AddElement(myfile())
myfile()\Host = PureHTTP_SplitURL(myurl.s,#PureHTTP_URL_TO_HOST)
myfile()\Path = PureHTTP_SplitURL(myurl.s,#PureHTTP_URL_TO_PATH)
myfile()\Destbuffer = AllocateMemory(#PureHTTP_Buffersize)
myfile()\limitspeed = 0
myurl.s = "http://rss.rssad.jp/rss/impresswatch/pcwatch.rdf"
AddElement(myfile())
myfile()\Host = PureHTTP_SplitURL(myurl.s,#PureHTTP_URL_TO_HOST)
myfile()\Path = PureHTTP_SplitURL(myurl.s,#PureHTTP_URL_TO_PATH)
myfile()\Destbuffer = AllocateMemory(#PureHTTP_Buffersize)
myfile()\limitspeed = 0
myurl.s = "http://av.watch.impress.co.jp/sublink/av.rdf"
AddElement(myfile())
myfile()\Host = PureHTTP_SplitURL(myurl.s,#PureHTTP_URL_TO_HOST)
myfile()\Path = PureHTTP_SplitURL(myurl.s,#PureHTTP_URL_TO_PATH)
myfile()\Destbuffer = AllocateMemory(#PureHTTP_Buffersize)
;myfile()\Callback = @MyCallback()
myfile()\limitspeed = 0
;// Start Download
Status.l = PureHTTP_Get_File(myfile())
;// Select Status after we are done.
Select Status.l
Case #PureHTTP_STATUS_CONNECTING ; Connecting
Case #PureHTTP_STATUS_CONNECTED ; Connected
Case #PureHTTP_STATUS_RECEIVE ; Receiving...
Case #PureHTTP_STATUS_IDLE ; Idle, Waiting for Data...
Case #PureHTTP_STATUS_FINISHED ; Download Finished
Debug "Finish"
Case #PureHTTP_STATUS_TIMEOUT ; Timeout
Debug "Timeout"
; PrintN("* Connection Timeout!")
Case #PureHTTP_STATUS_HTTPERROR ; HTTP Error / Return
; PrintN("HTTP ERROR "+myfile\Header\HTTPReturnString)
Debug "HTTP ERROR "+myfile()\Header\HTTPReturnString
Case #PureHTTP_STATUS_REDIRECT
; PrintN("HTTP REDIRECT 302")
Debug "302"
Case #PureHTTP_STATUS_FILEERROR ; Cant create file
; PrintN("Unable to Write File: "+myfile\outputfile)
Debug "FILE ERROR"
Case #PureHTTP_STATUS_ABORT ; Download Aborted
Debug "ABORT"
EndSelect
l=0
ForEach myfile()
;// !!! Debug Download to Memory Buffer
If myfile()\Destbuffer
Debug "Memorycontent:"
Debug "Memorybuffersize: "+Str(MemorySize(myfile()\Destbuffer))+" bytes"
Debug "Received Data : "+Str(myfile()\Totalbytes)+" bytes"
Debug PeekS(myfile()\Destbuffer)
EndIf
l+1
CreateFile(1, "mem"+Str(l)+".txt")
WriteData(1, myfile()\Destbuffer, myfile()\totalbytes)
CloseFile(1)
Next
