ReceiveHTTPToMemory() with Proxy support

infratec · Post by **infratec** » Thu Jul 07, 2016 11:29 am

IncludeFile "libcurl.pbi" ; https://github.com/deseven/pbsamples/tree/master/crossplatform/libcurl


CompilerIf #PB_Compiler_IsMainFile
  EnableExplicit
CompilerEndIf


Global *ReceiveHTTPToMemoryBuffer, ReceiveHTTPToMemoryBufferPtr.i


ProcedureC ReceiveHTTPWriteToMemoryFunction(*ptr, Size.i, NMemB.i, *Stream)
  
  Protected SizeProper.i  = Size & 255
  Protected NMemBProper.i = NMemB
  
  
  If *ReceiveHTTPToMemoryBuffer = 0
    *ReceiveHTTPToMemoryBuffer = AllocateMemory(SizeProper * NMemBProper)
  Else
    *ReceiveHTTPToMemoryBuffer = ReAllocateMemory(*ReceiveHTTPToMemoryBuffer, MemorySize(*ReceiveHTTPToMemoryBuffer) + SizeProper * NMemBProper)
  EndIf
  
  CopyMemory(*ptr, *ReceiveHTTPToMemoryBuffer + ReceiveHTTPToMemoryBufferPtr, SizeProper * NMemBProper)
  ReceiveHTTPToMemoryBufferPtr + SizeProper * NMemBProper
  
  ProcedureReturn SizeProper * NMemBProper
  
EndProcedure



Procedure.i ReceiveHTTPToMemory(URL$, Proxy$="")
  
  Protected *Buffer, curl.i, Timeout.i, res.i
  
  If Len(URL$)
    
    curl  = curl_easy_init()
    If curl
      
      Timeout = 3
      
      curl_easy_setopt(curl, #CURLOPT_URL, str2curl(URL$))
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
      curl_easy_setopt(curl, #CURLOPT_HEADER, @"")      
      curl_easy_setopt(curl, #CURLOPT_TIMEOUT, Timeout)
      
      If Len(Proxy$)
        curl_easy_setopt(curl, #CURLOPT_PROXY, @proxy$)
      EndIf
      
      curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @ReceiveHTTPWriteToMemoryFunction())
      res = curl_easy_perform(curl)
      
      If res = #CURLE_OK
        *Buffer = AllocateMemory(ReceiveHTTPToMemoryBufferPtr)
        If *Buffer
          CopyMemory(*ReceiveHTTPToMemoryBuffer, *Buffer, ReceiveHTTPToMemoryBufferPtr)
          FreeMemory(*ReceiveHTTPToMemoryBuffer)
          *ReceiveHTTPToMemoryBuffer = #Null
          ReceiveHTTPToMemoryBufferPtr = 0
        EndIf
      EndIf
      
      curl_easy_cleanup(curl)
      
    EndIf
    
  EndIf
  
  ProcedureReturn *Buffer
  
EndProcedure


CompilerIf #PB_Compiler_IsMainFile
  
  Define *Buffer, Proxy$
  
  InitNetwork()
  
  Proxy$ = InputRequester("ProxyServer", "Do you use a ProxyServer? Then enter the full url:", "")

  *Buffer = ReceiveHTTPToMemory("http://www.purebasic.fr/english/index.php", Proxy$)
  If *Buffer
    ShowMemoryViewer(*Buffer, MemorySize(*Buffer))
    FreeMemory(*Buffer)
  EndIf
  
CompilerEndIf

You have to enter the full url of the proxy:

http://test.proxy.xyz:8080

Bernd

djes · Post by **djes** » Mon Jul 11, 2016 10:44 am

Thank you for the code. I've tried to implement proxy user and password.

Code: Select all

IncludeFile "libcurl.pbi" ; https://github.com/deseven/pbsamples/tree/master/crossplatform/libcurl


CompilerIf #PB_Compiler_IsMainFile
  EnableExplicit
CompilerEndIf


Global *ReceiveHTTPToMemoryBuffer, ReceiveHTTPToMemoryBufferPtr.i


ProcedureC ReceiveHTTPWriteToMemoryFunction(*ptr, Size.i, NMemB.i, *Stream)
  
  Protected SizeProper.i  = Size & 255
  Protected NMemBProper.i = NMemB
  
  If *ReceiveHTTPToMemoryBuffer = 0
    *ReceiveHTTPToMemoryBuffer = AllocateMemory(SizeProper * NMemBProper)
    If *ReceiveHTTPToMemoryBuffer = 0
      Debug "Problem allocating memory"
      End
    EndIf
  Else
    *ReceiveHTTPToMemoryBuffer = ReAllocateMemory(*ReceiveHTTPToMemoryBuffer, MemorySize(*ReceiveHTTPToMemoryBuffer) + SizeProper * NMemBProper)
    If *ReceiveHTTPToMemoryBuffer = 0
      Debug "Problem reallocating memory"
      End
    EndIf  
  EndIf
  
  CopyMemory(*ptr, *ReceiveHTTPToMemoryBuffer + ReceiveHTTPToMemoryBufferPtr, SizeProper * NMemBProper)
  ReceiveHTTPToMemoryBufferPtr + SizeProper * NMemBProper
  
  ProcedureReturn SizeProper * NMemBProper
  
EndProcedure



Procedure.i ReceiveHTTPToMemory(URL$, ProxyURL$="", ProxyPort$="", ProxyUser$="", ProxyPassword$="")
  
  Protected *Buffer, curl.i, Timeout.i, res.i
  
  If Len(URL$)
    
    curl  = curl_easy_init()
    If curl
      
      Timeout = 3
      
      curl_easy_setopt(curl, #CURLOPT_URL, str2curl(URL$))
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
      curl_easy_setopt(curl, #CURLOPT_HEADER, 0) ;No headers, please :)
      curl_easy_setopt(curl, #CURLOPT_TIMEOUT, Timeout)
      
      If Len(ProxyURL$)
        ;curl_easy_setopt(curl, #CURLOPT_HTTPPROXYTUNNEL, #True)
        If Len(ProxyPort$)
          ProxyURL$ + ":" + ProxyPort$
        EndIf
        ;Debug ProxyURL$
        curl_easy_setopt(curl, #CURLOPT_PROXY, str2curl(ProxyURL$))
        If Len(ProxyUser$)
          If Len(ProxyPassword$)
            ProxyUser$ + ":" + ProxyPassword$
          EndIf
          ;Debug ProxyUser$
          curl_easy_setopt(curl, #CURLOPT_PROXYUSERPWD, str2curl(ProxyUser$))
        EndIf
      EndIf
      
      curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @ReceiveHTTPWriteToMemoryFunction())
      res = curl_easy_perform(curl)
      
      If res = #CURLE_OK
        *Buffer = AllocateMemory(ReceiveHTTPToMemoryBufferPtr)
        If *Buffer
          CopyMemory(*ReceiveHTTPToMemoryBuffer, *Buffer, ReceiveHTTPToMemoryBufferPtr)
          FreeMemory(*ReceiveHTTPToMemoryBuffer)
          *ReceiveHTTPToMemoryBuffer = #Null
          ReceiveHTTPToMemoryBufferPtr = 0
        Else
          Debug "Problem allocating buffer"         
        EndIf        
        curl_easy_cleanup(curl)
      Else
        Debug "CURL NOT OK"
      EndIf
      
    Else
      Debug "Can't Init CURL"
    EndIf
    
  EndIf
  
  ProcedureReturn *Buffer
  
EndProcedure


CompilerIf #PB_Compiler_IsMainFile
  
  Define *Buffer, ProxyURL$, ProxyPort$, ProxyUser$, ProxyPass$
  
  InitNetwork()
  
  ProxyURL$  = InputRequester("ProxyServer", "Do you use a Proxy Server? Then enter the full url:", "")
  ProxyPort$ = InputRequester("ProxyPort"  , "Do you use a specific port? Then enter it", "")
  ProxyUser$ = InputRequester("ProxyUser"  , "Do you use a user name? Then enter it", "")
  ProxyPass$ = InputRequester("ProxyPass"  , "Do you use a password? Then enter it:", "")
  
  *Buffer = ReceiveHTTPToMemory("http://www.purebasic.fr/english/index.php", ProxyURL$, ProxyPort$, ProxyUser$, ProxyPass$)
  If *Buffer
    ShowMemoryViewer(*Buffer, MemorySize(*Buffer))
    FreeMemory(*Buffer)
  Else
    MessageRequester("Erreur", "Problem while loading", #PB_MessageRequester_Ok )
  EndIf
  
CompilerEndIf

Edit: modified the line 54 to not get headers.

Kwai chang caine · Post by **Kwai chang caine** » Mon Jul 11, 2016 1:41 pm

Thanks a lot at you two for your code.
But like always...that not works for me

I have a proxy with login and Password authentification, and it return "This method is not allowed" with the code of DJES anf nothing with the code of INFRATEC "*Buffer = 0"

and

result: 7

with the test code

Code: Select all

; https://raw.githubusercontent.com/deseven/pbsamples/master/crossplatform/libcurl/curl_example.pb

; working with static libcurl
InitNetwork()
IncludeFile "libcurl.pbi"

curl = curl_easy_init()
url.s = str2curl("http://deseven.info/test.php")
agent.s = str2curl("pbcurl/1.0")
cookie.s = str2curl("var=value;")
;post.s = str2curl("login=mylogin&password=mypassword")
header.s = str2curl("Cache-Control: no-cache")
If curl
  curl_easy_setopt(curl,#CURLOPT_URL,@url)
  curl_easy_setopt(curl,#CURLOPT_IPRESOLVE,#CURL_IPRESOLVE_V4)
  curl_easy_setopt(curl,#CURLOPT_COOKIE,@cookie)
  curl_easy_setopt(curl,#CURLOPT_POSTFIELDS,@post)
  curl_easy_setopt(curl,#CURLOPT_USERAGENT,@agent)
  curl_easy_setopt(curl,#CURLOPT_TIMEOUT,30)
  *header = curl_slist_append(0,header)
  curl_easy_setopt(curl,#CURLOPT_HTTPHEADER,*header)
  curl_easy_setopt(curl,#CURLOPT_WRITEFUNCTION,@curlWriteData())
  res = curl_easy_perform(curl)
  resData.s = curlGetData()
  curl_easy_getinfo(curl,#CURLINFO_RESPONSE_CODE,@resHTTP)
  Debug "result: " + Str(res)
  If Not res
    Debug "HTTP code: " + Str(resHTTP)
    Debug "HTTP data: " + #CRLF$ + resData
  EndIf
  curl_easy_cleanup(curl)
Else
  Debug "can't init curl!"
EndIf

djes · Post by **djes** » Mon Jul 11, 2016 3:21 pm

Try to uncomment the line 58 : curl_easy_setopt(curl, #CURLOPT_HTTPPROXYTUNNEL, #True)

Kwai chang caine · Post by **Kwai chang caine** » Tue Jul 12, 2016 3:42 pm

Hello DJES

I have commented the line like you say and i have the result

CURL NOT OK

Else, if i enter manually my Login/Password, and after run your code that works

Apparently, my Login/Password is not sending correctly to the PROXY with your code

infratec · Post by **infratec** » Thu Jul 14, 2016 10:15 pm

Maybe you have to set the type of authentication.

Look here:

https://curl.haxx.se/libcurl/c/CURLOPT_PROXYAUTH.html

And here:

https://curl.haxx.se/libcurl/c/CURLOPT_HTTPAUTH.html

Bernd

djes · Post by **djes** » Mon Jul 18, 2016 9:03 am

Infratec is right. There's several way of authenticating, all implemented by curl easily (it's way this lib is great). But you have to handle them yourself. Maybe you could do a little select box to choose and test each of them. For example, code for CURLAUTH_BASIC is

Code: Select all

curl_easy_setopt(curl, #CURLOPT_PROXYAUTH, #CURLAUTH_BASIC)

thyphoon · Post by **thyphoon** » Tue Jul 19, 2016 8:33 am

Hello ! Thanks to infratec for this great code !

I find a hook to replace orginal ReceiveHTTPMemory by your curl wrapper and the Djes ReceiveHTTPToMemory procedure to support Proxy
Know you just add this code with a include and use at the start

Code: Select all

libcurl::InitProxy(ProxyURL$ ,ProxyPort$,ProxyUser$,ProxyPass$)

to setup proxy
and after you can use normaly ReceiveHTTPMemory(URL$)

Code: Select all


DeclareModule libcurl
  Declare InitProxy( ProxyURL.s,ProxyPort.s,ProxyUser.s,ProxyPass.s)
  Declare.i ReceiveHTTPToMemory(URL$)
EndDeclareModule  

Module libcurl
  
  IncludeFile "libcurl.pbi" ; https://github.com/deseven/pbsamples/tree/master/crossplatform/libcurl
  
  Structure http_curl_ext
    ProxyURL.s
    ProxyPort.s
    ProxyUser.s
    ProxyPass.s
  EndStructure
  
  Global http.http_curl_ext
  
  Procedure InitProxy( ProxyURL.s,ProxyPort.s,ProxyUser.s,ProxyPass.s)
    http\ProxyURL=ProxyURL
    http\ProxyPort=ProxyPort
    http\ProxyUser=ProxyUser
    http\ProxyPass=ProxyPass
  EndProcedure
  
  Global *ReceiveHTTPToMemoryBuffer, ReceiveHTTPToMemoryBufferPtr.i
  
  
ProcedureC ReceiveHTTPWriteToMemoryFunction(*ptr, Size.i, NMemB.i, *Stream)
  
  Protected SizeProper.i  = Size & 255
  Protected NMemBProper.i = NMemB
  
  If *ReceiveHTTPToMemoryBuffer = 0
    *ReceiveHTTPToMemoryBuffer = AllocateMemory(SizeProper * NMemBProper)
    If *ReceiveHTTPToMemoryBuffer = 0
      Debug "Problem allocating memory"
      End
    EndIf
  Else
    *ReceiveHTTPToMemoryBuffer = ReAllocateMemory(*ReceiveHTTPToMemoryBuffer, MemorySize(*ReceiveHTTPToMemoryBuffer) + SizeProper * NMemBProper)
    If *ReceiveHTTPToMemoryBuffer = 0
      Debug "Problem reallocating memory"
      End
    EndIf  
  EndIf
  
  CopyMemory(*ptr, *ReceiveHTTPToMemoryBuffer + ReceiveHTTPToMemoryBufferPtr, SizeProper * NMemBProper)
  ReceiveHTTPToMemoryBufferPtr + SizeProper * NMemBProper
  
  ProcedureReturn SizeProper * NMemBProper
  
EndProcedure



Procedure.i ReceiveHTTPToMemory(URL$)
  ProxyURL$=http\ProxyURL
  ProxyPort$=http\ProxyPort
  ProxyUser$=http\ProxyUser
  ProxyPassword$=http\ProxyPass
  
  Protected *Buffer, curl.i, Timeout.i, res.i
  
  If Len(URL$)
    
    curl  = curl_easy_init()
    If curl
      
      Timeout = 3
      
      curl_easy_setopt(curl, #CURLOPT_URL, str2curl(URL$))
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
      curl_easy_setopt(curl, #CURLOPT_HEADER, 0) ;No headers, please :)
      curl_easy_setopt(curl, #CURLOPT_TIMEOUT, Timeout)
      
      If Len(ProxyURL$)
        ;curl_easy_setopt(curl, #CURLOPT_HTTPPROXYTUNNEL, #True)
        If Len(ProxyPort$)
          ProxyURL$ + ":" + ProxyPort$
        EndIf
        ;Debug ProxyURL$
        curl_easy_setopt(curl, #CURLOPT_PROXY, str2curl(ProxyURL$))
        If Len(ProxyUser$)
          If Len(ProxyPassword$)
            ProxyUser$ + ":" + ProxyPassword$
          EndIf
          ;Debug ProxyUser$
          curl_easy_setopt(curl, #CURLOPT_PROXYUSERPWD, str2curl(ProxyUser$))
        EndIf
      EndIf
      
      curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @ReceiveHTTPWriteToMemoryFunction())
      res = curl_easy_perform(curl)
      
      If res = #CURLE_OK
        *Buffer = AllocateMemory(ReceiveHTTPToMemoryBufferPtr)
        If *Buffer
          CopyMemory(*ReceiveHTTPToMemoryBuffer, *Buffer, ReceiveHTTPToMemoryBufferPtr)
          FreeMemory(*ReceiveHTTPToMemoryBuffer)
          *ReceiveHTTPToMemoryBuffer = #Null
          ReceiveHTTPToMemoryBufferPtr = 0
        Else
          Debug "Problem allocating buffer"         
        EndIf        
        curl_easy_cleanup(curl)
      Else
        Debug "CURL NOT OK"
      EndIf
      
    Else
      Debug "Can't Init CURL"
    EndIf
    
  EndIf
  
  ProcedureReturn *Buffer
  
  
EndProcedure
  
EndModule


;-HOOK to remplace original ReceiveHTTPMemory
Macro ReceiveHTTPMemory(URL)
libcurl::ReceiveHTTPToMemory(URL):MessageRequester("coucou","coucou")
EndMacro  

CompilerIf #PB_Compiler_IsMainFile
 
  InitNetwork()
  ProxyURL$  = InputRequester("ProxyServer", "Do you use a Proxy Server? Then enter the full url:", "")
  ProxyPort$ = InputRequester("ProxyPort"  , "Do you use a specific port? Then enter it", "")
  ProxyUser$ = InputRequester("ProxyUser"  , "Do you use a user name? Then enter it", "")
  ProxyPass$ = InputRequester("ProxyPass"  , "Do you use a password? Then enter it:", "")
  libcurl::InitProxy(ProxyURL$ ,ProxyPort$,ProxyUser$,ProxyPass$)

  *Buffer = ReceiveHTTPMemory("http://www.purebasic.fr/english/index.php")
  If *Buffer
    ShowMemoryViewer(*Buffer, MemorySize(*Buffer))
    FreeMemory(*Buffer)
  Else
    MessageRequester("Erreur", "Problem while loading", #PB_MessageRequester_Ok )
  EndIf
  
CompilerEndIf

novablue · Post by **novablue** » Tue Jun 20, 2017 9:20 pm

How do i make this work with socks5 proxy?

PureBasic Forums - English

ReceiveHTTPToMemory() with Proxy support

ReceiveHTTPToMemory() with Proxy support

Re: ReceiveHTTPToMemory() with Proxy support

Re: ReceiveHTTPToMemory() with Proxy support

Re: ReceiveHTTPToMemory() with Proxy support

Re: ReceiveHTTPToMemory() with Proxy support

Re: ReceiveHTTPToMemory() with Proxy support

Re: ReceiveHTTPToMemory() with Proxy support

Re: ReceiveHTTPToMemory() with Proxy support

Re: ReceiveHTTPToMemory() with Proxy support