HTTP Get Request Problem

Killswitch · Post by **Killswitch** » Sat Sep 09, 2006 9:16 pm

I'm experiementing with writing a crawler that'll check for broken links etc on pages.

Anyway, I used a HTTP sniffer to record the exact packet sent via IE to make sure the GET request I'm sending was correct. Here's my code:

Code: Select all

InitNetwork()

#CRLF=Chr(13)+Chr(10)

String.s="GET /~douglas.marsh/about.htm HTTP/1.1"+#CRLF
String+"Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"
String+"Accept-Language: en-gb"+#CRLF
String+"Accept-Encoding: gzip, deflate"+#CRLF
String+"If-Modified-Since: Mon, 21 Mar 2005 16:20:14 GMT"+#CRLF
String+"If-None-Match: "+Chr(34)+"ac0ae9-1429-423ef43e"+Chr(34)+#CRLF
String+"User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2)"+#CRLF
String+"Host: www.btinternet.com"+#CRLF
String+"Connection: Keep-Alive"

Connection=OpenNetworkConnection("www.btinernet.com",80)

If Connection
  SendNetworkString(Connection,String)
  Repeat
    Select NetworkClientEvent(Connection)
      
      Case 5
        Debug 3
      
      Case 3
        Debug 2
      
      Case #PB_NetworkEvent_Data
        Debug 1
    EndSelect
  ForEver
Else
  Debug -1
EndIf

It appears to connect, but I don't seem to get any response. Can anyone help?

Lep · Post by **Lep** » Sat Sep 09, 2006 10:45 pm

You need to add two crlfs after the last part of your header (String+"Connection: Keep-Alive"+#CRLF+#CRLF), then the server will send the response. To get the response, use something like the following under Case #PB_NetworkEvent_Data:

Code: Select all

len.l = 1
str.s = ""
While len
  buffer.l = AllocateMemory(1024)
  len = ReceiveNetworkData(Connection, buffer, 1024)
  str + PeekS(buffer, len)
  FreeMemory(buffer)
Wend
Debug str

Killswitch · Post by **Killswitch** » Sat Sep 09, 2006 10:48 pm

Wow, that was ridiculously simple. Thank you!

DarkDragon · Post by **DarkDragon** » Sun Sep 10, 2006 7:50 am

Code: Select all

; HTTP Commands - DarkDragon
; ***************************
; Commands:
; ---------
; HTTPRequest(URL.s, *EndSize.LONG, PacketSize, *Callback) - Result: Pointer to a buffer containing the webdata
; URL.s       - [in]  Nullterminated string with the link to the file
; *EndSize    - [out] Pointer to a long variable which receives the size of the resulting buffer
; PacketSize  - [in]  Long variable containing the size of the packets(steps) to download
; *Callback   - [in]  Pointer to a callback method receiving the following parameters: CurrentDownloadedSize.l, MaximumLength.l

; HTTPRequest_Password(URL.s, *EndSize.LONG, PacketSize, Username.s, Password.s, *Callback) - Result: Pointer to a buffer containing the webdata
; URL.s       - [in]  Nullterminated string with the link to the file
; *EndSize    - [out] Pointer to a long variable which receives the size of the resulting buffer
; PacketSize  - [in]  Long variable containing the size of the packets(steps) to download
; Username.s  - [in]  Nullterminated string containing the username
; Password.s  - [in]  Nullterminated string containing the password(NOT encoded)
; *Callback   - [in]  Pointer to a callback method receiving the following parameters: CurrentDownloadedSize.l, MaximumLength.l

InitNetwork()

#TIMEOUT = 1500

Procedure.s ReceiveLine(ConnectionID)
  Text.s = ""
  While char.b <> #LF
    char = 0
    ReceiveNetworkData(ConnectionID, @char, 1)
    If char <> 0
      Text.s + Chr(char)
    EndIf
  Wend
  
  ProcedureReturn RemoveString(RemoveString(Text, #CR$), #LF$)
EndProcedure

Procedure HTTPRequest(URL.s, *EndSize.LONG, PacketSize, *Callback)
  Protected Size.l, Data_.s, s.l, Method.s, File.s, Content.s, Text.s, Length.l, Line.s
  Protected CurSize.l, oSize.l, t.l, ConnectionID.l
  
  If Left(URL, 7) = "http://" : URL = Right(URL, Len(URL)-7) : EndIf
  s = FindString(URL, "/", 1)
  Host.s = "" : File.s = ""
  If s <> 0 : Host.s = Left(URL, s-1) : File.s = Right(URL, Len(URL)-s) : Else : Host = URL : EndIf
  
  s = FindString(File, "?", 1)
  If s <> 0
    Method.s = "POST"
    Content.s = Right(File, Len(File)-s)
    File = Left(File, s-1)
  Else
    Method.s = "GET"
    Content.s = ""
  EndIf
  
  ConnectionID = OpenNetworkConnection(Host, 80)
  If ConnectionID
    Data_.s = Method+" /"+File+" HTTP/1.0"+#CRLF$
    Data_.s + "Host: "+Host+#CRLF$
    Data_.s + "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.12) Gecko/20050915 Firefox/1.0.7"+#CRLF$
    If Content <> ""
      Data_.s + "Connection: close"+#CRLF$
      Data_.s + "Content-Type: application/x-www-form-urlencoded"+#CRLF$
      Data_.s + "Content-Length: "+Str(Len(Content))+#CRLF$
      Data_.s + #CRLF$ + Content.s + #CRLF$+#CRLF$
    Else
      Data_.s + "Connection: close"+#CRLF$+#CRLF$
    EndIf
    
    SendNetworkString(ConnectionID, Data_)
    While NetworkClientEvent(ConnectionID) <> 2 : Delay(10) : Wend
    
    Length = 0
    Line.s = ""
    Repeat
      Line = ReceiveLine(ConnectionID)
      Select LCase(StringField(Line, 1, ":"))
        Case "content-length"
          
          Length = Val(Trim(StringField(Line, 2, ":")))
      EndSelect
    Until Len(Trim(Line)) <= 4
    
    Size = 0
    If Length <> 0
      
      *Result = AllocateMemory(Length)
      
      While Size < Length
        If CurSize > 0
          Size + CurSize
          If *Callback
            CallFunctionFast(*Callback, Size, Length)
          EndIf
        EndIf
        If Size > (Length-PacketSize)
          PacketSize = Length-Size
        EndIf
        If PacketSize > 0
        CurSize = ReceiveNetworkData(ConnectionID, *Result+Size, PacketSize)
        EndIf
      Wend
      
    Else
      
      *Buffer = AllocateMemory(PacketSize)
      *Result = AllocateMemory(1)
      t = ElapsedMilliseconds()
      While ElapsedMilliseconds()-t <= #TIMEOUT
        If NetworkClientEvent(ConnectionID) = 2
          CurSize = ReceiveNetworkData(ConnectionID, *Buffer, PacketSize)
          If CurSize > 0
            oSize = Size
            Size + CurSize
            *Result = ReAllocateMemory(*Result, Size)
            CopyMemory(*Buffer, *Result+oSize, CurSize)
            If *Callback
              CallFunctionFast(*Callback, Size, 0)
            EndIf
          EndIf
          t = ElapsedMilliseconds()
        EndIf
      Wend
      Length = Size
      
    EndIf
    
    *EndSize\l = Length
    If *Callback
      CallFunctionFast(*Callback, Size, Length)
    EndIf
    
    CloseNetworkConnection(ConnectionID)
    
    ProcedureReturn *Result
  EndIf
EndProcedure

Procedure HTTPRequest_Password(URL.s, *EndSize.LONG, PacketSize, Username.s, Password.s, *Callback)
  Protected Size.l, Data_.s, s.l, Method.s, File.s, Content.s, Text.s, Length.l, Line.s
  Protected CurSize.l, oSize.l, t.l, ConnectionID.l
  
  If Left(URL, 7) = "http://" : URL = Right(URL, Len(URL)-7) : EndIf
  s = FindString(URL, "/", 1)
  Host.s = "" : File.s = ""
  If s <> 0 : Host.s = Left(URL, s-1) : File.s = Right(URL, Len(URL)-s) : Else : Host = URL : EndIf
  
  s = FindString(File, "?", 1)
  If s <> 0
    Method.s = "POST"
    Content.s = Right(File, Len(File)-s)
    File = Left(File, s-1)
  Else
    Method.s = "GET"
    Content.s = ""
  EndIf
  
  InputBuffer.s = Username+":"+Password
  OutputBuffer.s = Space(256)
  Base64Encoder(@InputBuffer, Len(InputBuffer), @OutputBuffer, 255)
  
  ConnectionID = OpenNetworkConnection(Host, 80)
  If ConnectionID
    Data_.s = Method+" /"+File+" HTTP/1.0" + #CRLF$
    Data_.s + "Host: "+Host + #CRLF$
    Data_.s + "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.12) Gecko/20050915 Firefox/1.0.7" + #CRLF$
    Data_.s + "Authorization: Basic "+ OutputBuffer + #CRLF$
    If Content <> ""
      Data_.s + "Connection: close" + #CRLF$
      Data_.s + "Content-Type: application/x-www-form-urlencoded"+#CRLF$
      Data_.s + "Content-Length: "+Str(Len(Content))+#CRLF$
      Data_.s + #CRLF$ + Content.s + #CRLF$+#CRLF$
    Else
      Data_.s + "Connection: close" + #CRLF$ + #CRLF$
    EndIf
    
    SendNetworkString(ConnectionID, Data_)
    While NetworkClientEvent(ConnectionID) <> 2 : Delay(10) : Wend
    
    Length = 0
    Line.s = ""
    Repeat
      Line = ReceiveLine(ConnectionID)
      Select LCase(StringField(Line, 1, ":"))
        Case "content-length"
          
          Length = Val(Trim(StringField(Line, 2, ":")))
      EndSelect
    Until Len(Trim(Line)) <= 4
    
    Size = 0
    If Length <> 0
      
      *Result = AllocateMemory(Length)
      
      While Size < Length
        If CurSize > 0
          Size + CurSize
          If *Callback
            CallFunctionFast(*Callback, Size, Length)
          EndIf
        EndIf
        If Size > (Length-PacketSize)
          PacketSize = Length-Size
        EndIf
        If PacketSize > 0
        CurSize = ReceiveNetworkData(ConnectionID, *Result+Size, PacketSize)
        EndIf
      Wend
      
    Else
      
      *Buffer = AllocateMemory(PacketSize)
      *Result = AllocateMemory(1)
      t = ElapsedMilliseconds()
      While ElapsedMilliseconds()-t <= #TIMEOUT
        If NetworkClientEvent(ConnectionID) = 2
          CurSize = ReceiveNetworkData(ConnectionID, *Buffer, PacketSize)
          If CurSize > 0
            oSize = Size
            Size + CurSize
            *Result = ReAllocateMemory(*Result, Size)
            CopyMemory(*Buffer, *Result+oSize, CurSize)
            If *Callback
              CallFunctionFast(*Callback, Size, 0)
            EndIf
          EndIf
          t = ElapsedMilliseconds()
        EndIf
      Wend
      Length = Size
      
    EndIf
    
    *EndSize\l = Length
    If *Callback
      CallFunctionFast(*Callback, Size, Length)
    EndIf
    
    CloseNetworkConnection(ConnectionID)
    
    ProcedureReturn *Result
  EndIf
EndProcedure

; Example:

; *Buffer = HTTPRequest("http://www.bradan.net/", @Size, 16, 0)
; Debug Size
; Debug PeekS(*Buffer, Size)
; FreeMemory(*Buffer)

It's free

.