PureBasic Forums - English

http://www.purebasic.com
https://www.purebasic.fr/english/

ReceiveHTTPToMemory() with Proxy support

https://www.purebasic.fr/english/viewtopic.php?t=66099

Page 1 of 1

ReceiveHTTPToMemory() with Proxy support

Posted: Thu Jul 07, 2016 11:29 am
by infratec

Code: Select all

IncludeFile "libcurl.pbi" ; https://github.com/deseven/pbsamples/tree/master/crossplatform/libcurl


CompilerIf #PB_Compiler_IsMainFile
  EnableExplicit
CompilerEndIf


Global *ReceiveHTTPToMemoryBuffer, ReceiveHTTPToMemoryBufferPtr.i


ProcedureC ReceiveHTTPWriteToMemoryFunction(*ptr, Size.i, NMemB.i, *Stream)
  
  Protected SizeProper.i  = Size & 255
  Protected NMemBProper.i = NMemB
  
  
  If *ReceiveHTTPToMemoryBuffer = 0
    *ReceiveHTTPToMemoryBuffer = AllocateMemory(SizeProper * NMemBProper)
  Else
    *ReceiveHTTPToMemoryBuffer = ReAllocateMemory(*ReceiveHTTPToMemoryBuffer, MemorySize(*ReceiveHTTPToMemoryBuffer) + SizeProper * NMemBProper)
  EndIf
  
  CopyMemory(*ptr, *ReceiveHTTPToMemoryBuffer + ReceiveHTTPToMemoryBufferPtr, SizeProper * NMemBProper)
  ReceiveHTTPToMemoryBufferPtr + SizeProper * NMemBProper
  
  ProcedureReturn SizeProper * NMemBProper
  
EndProcedure



Procedure.i ReceiveHTTPToMemory(URL$, Proxy$="")
  
  Protected *Buffer, curl.i, Timeout.i, res.i
  
  If Len(URL$)
    
    curl  = curl_easy_init()
    If curl
      
      Timeout = 3
      
      curl_easy_setopt(curl, #CURLOPT_URL, str2curl(URL$))
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
      curl_easy_setopt(curl, #CURLOPT_HEADER, @"")      
      curl_easy_setopt(curl, #CURLOPT_TIMEOUT, Timeout)
      
      If Len(Proxy$)
        curl_easy_setopt(curl, #CURLOPT_PROXY, @proxy$)
      EndIf
      
      curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @ReceiveHTTPWriteToMemoryFunction())
      res = curl_easy_perform(curl)
      
      If res = #CURLE_OK
        *Buffer = AllocateMemory(ReceiveHTTPToMemoryBufferPtr)
        If *Buffer
          CopyMemory(*ReceiveHTTPToMemoryBuffer, *Buffer, ReceiveHTTPToMemoryBufferPtr)
          FreeMemory(*ReceiveHTTPToMemoryBuffer)
          *ReceiveHTTPToMemoryBuffer = #Null
          ReceiveHTTPToMemoryBufferPtr = 0
        EndIf
      EndIf
      
      curl_easy_cleanup(curl)
      
    EndIf
    
  EndIf
  
  ProcedureReturn *Buffer
  
EndProcedure


CompilerIf #PB_Compiler_IsMainFile
  
  Define *Buffer, Proxy$
  
  InitNetwork()
  
  Proxy$ = InputRequester("ProxyServer", "Do you use a ProxyServer? Then enter the full url:", "")

  *Buffer = ReceiveHTTPToMemory("http://www.purebasic.fr/english/index.php", Proxy$)
  If *Buffer
    ShowMemoryViewer(*Buffer, MemorySize(*Buffer))
    FreeMemory(*Buffer)
  EndIf
  
CompilerEndIf
You have to enter the full url of the proxy:
Bernd

Re: ReceiveHTTPToMemory() with Proxy support

Posted: Mon Jul 11, 2016 10:44 am
by djes
Thank you for the code. I've tried to implement proxy user and password.

Code: Select all

IncludeFile "libcurl.pbi" ; https://github.com/deseven/pbsamples/tree/master/crossplatform/libcurl


CompilerIf #PB_Compiler_IsMainFile
  EnableExplicit
CompilerEndIf


Global *ReceiveHTTPToMemoryBuffer, ReceiveHTTPToMemoryBufferPtr.i


ProcedureC ReceiveHTTPWriteToMemoryFunction(*ptr, Size.i, NMemB.i, *Stream)
  
  Protected SizeProper.i  = Size & 255
  Protected NMemBProper.i = NMemB
  
  If *ReceiveHTTPToMemoryBuffer = 0
    *ReceiveHTTPToMemoryBuffer = AllocateMemory(SizeProper * NMemBProper)
    If *ReceiveHTTPToMemoryBuffer = 0
      Debug "Problem allocating memory"
      End
    EndIf
  Else
    *ReceiveHTTPToMemoryBuffer = ReAllocateMemory(*ReceiveHTTPToMemoryBuffer, MemorySize(*ReceiveHTTPToMemoryBuffer) + SizeProper * NMemBProper)
    If *ReceiveHTTPToMemoryBuffer = 0
      Debug "Problem reallocating memory"
      End
    EndIf  
  EndIf
  
  CopyMemory(*ptr, *ReceiveHTTPToMemoryBuffer + ReceiveHTTPToMemoryBufferPtr, SizeProper * NMemBProper)
  ReceiveHTTPToMemoryBufferPtr + SizeProper * NMemBProper
  
  ProcedureReturn SizeProper * NMemBProper
  
EndProcedure



Procedure.i ReceiveHTTPToMemory(URL$, ProxyURL$="", ProxyPort$="", ProxyUser$="", ProxyPassword$="")
  
  Protected *Buffer, curl.i, Timeout.i, res.i
  
  If Len(URL$)
    
    curl  = curl_easy_init()
    If curl
      
      Timeout = 3
      
      curl_easy_setopt(curl, #CURLOPT_URL, str2curl(URL$))
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
      curl_easy_setopt(curl, #CURLOPT_HEADER, 0) ;No headers, please :)
      curl_easy_setopt(curl, #CURLOPT_TIMEOUT, Timeout)
      
      If Len(ProxyURL$)
        ;curl_easy_setopt(curl, #CURLOPT_HTTPPROXYTUNNEL, #True)
        If Len(ProxyPort$)
          ProxyURL$ + ":" + ProxyPort$
        EndIf
        ;Debug ProxyURL$
        curl_easy_setopt(curl, #CURLOPT_PROXY, str2curl(ProxyURL$))
        If Len(ProxyUser$)
          If Len(ProxyPassword$)
            ProxyUser$ + ":" + ProxyPassword$
          EndIf
          ;Debug ProxyUser$
          curl_easy_setopt(curl, #CURLOPT_PROXYUSERPWD, str2curl(ProxyUser$))
        EndIf
      EndIf
      
      curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @ReceiveHTTPWriteToMemoryFunction())
      res = curl_easy_perform(curl)
      
      If res = #CURLE_OK
        *Buffer = AllocateMemory(ReceiveHTTPToMemoryBufferPtr)
        If *Buffer
          CopyMemory(*ReceiveHTTPToMemoryBuffer, *Buffer, ReceiveHTTPToMemoryBufferPtr)
          FreeMemory(*ReceiveHTTPToMemoryBuffer)
          *ReceiveHTTPToMemoryBuffer = #Null
          ReceiveHTTPToMemoryBufferPtr = 0
        Else
          Debug "Problem allocating buffer"         
        EndIf        
        curl_easy_cleanup(curl)
      Else
        Debug "CURL NOT OK"
      EndIf
      
    Else
      Debug "Can't Init CURL"
    EndIf
    
  EndIf
  
  ProcedureReturn *Buffer
  
EndProcedure


CompilerIf #PB_Compiler_IsMainFile
  
  Define *Buffer, ProxyURL$, ProxyPort$, ProxyUser$, ProxyPass$
  
  InitNetwork()
  
  ProxyURL$  = InputRequester("ProxyServer", "Do you use a Proxy Server? Then enter the full url:", "")
  ProxyPort$ = InputRequester("ProxyPort"  , "Do you use a specific port? Then enter it", "")
  ProxyUser$ = InputRequester("ProxyUser"  , "Do you use a user name? Then enter it", "")
  ProxyPass$ = InputRequester("ProxyPass"  , "Do you use a password? Then enter it:", "")
  
  *Buffer = ReceiveHTTPToMemory("http://www.purebasic.fr/english/index.php", ProxyURL$, ProxyPort$, ProxyUser$, ProxyPass$)
  If *Buffer
    ShowMemoryViewer(*Buffer, MemorySize(*Buffer))
    FreeMemory(*Buffer)
  Else
    MessageRequester("Erreur", "Problem while loading", #PB_MessageRequester_Ok )
  EndIf
  
CompilerEndIf
Edit: modified the line 54 to not get headers.

Re: ReceiveHTTPToMemory() with Proxy support

Posted: Mon Jul 11, 2016 1:41 pm
by Kwai chang caine
Thanks a lot at you two for your code.
But like always...that not works for me :oops:
I have a proxy with login and Password authentification, and it return "This method is not allowed" with the code of DJES anf nothing with the code of INFRATEC "*Buffer = 0" :(
and
result: 7
with the test code

Code: Select all

; https://raw.githubusercontent.com/deseven/pbsamples/master/crossplatform/libcurl/curl_example.pb

; working with static libcurl
InitNetwork()
IncludeFile "libcurl.pbi"

curl = curl_easy_init()
url.s = str2curl("http://deseven.info/test.php")
agent.s = str2curl("pbcurl/1.0")
cookie.s = str2curl("var=value;")
;post.s = str2curl("login=mylogin&password=mypassword")
header.s = str2curl("Cache-Control: no-cache")
If curl
  curl_easy_setopt(curl,#CURLOPT_URL,@url)
  curl_easy_setopt(curl,#CURLOPT_IPRESOLVE,#CURL_IPRESOLVE_V4)
  curl_easy_setopt(curl,#CURLOPT_COOKIE,@cookie)
  curl_easy_setopt(curl,#CURLOPT_POSTFIELDS,@post)
  curl_easy_setopt(curl,#CURLOPT_USERAGENT,@agent)
  curl_easy_setopt(curl,#CURLOPT_TIMEOUT,30)
  *header = curl_slist_append(0,header)
  curl_easy_setopt(curl,#CURLOPT_HTTPHEADER,*header)
  curl_easy_setopt(curl,#CURLOPT_WRITEFUNCTION,@curlWriteData())
  res = curl_easy_perform(curl)
  resData.s = curlGetData()
  curl_easy_getinfo(curl,#CURLINFO_RESPONSE_CODE,@resHTTP)
  Debug "result: " + Str(res)
  If Not res
    Debug "HTTP code: " + Str(resHTTP)
    Debug "HTTP data: " + #CRLF$ + resData
  EndIf
  curl_easy_cleanup(curl)
Else
  Debug "can't init curl!"
EndIf

Re: ReceiveHTTPToMemory() with Proxy support

Posted: Mon Jul 11, 2016 3:21 pm
by djes
Try to uncomment the line 58 : curl_easy_setopt(curl, #CURLOPT_HTTPPROXYTUNNEL, #True)

Re: ReceiveHTTPToMemory() with Proxy support

Posted: Tue Jul 12, 2016 3:42 pm
by Kwai chang caine
Hello DJES :D
I have commented the line like you say and i have the result
CURL NOT OK
Else, if i enter manually my Login/Password, and after run your code that works :shock:
Apparently, my Login/Password is not sending correctly to the PROXY with your code :(

Re: ReceiveHTTPToMemory() with Proxy support

Posted: Thu Jul 14, 2016 10:15 pm
by infratec
Maybe you have to set the type of authentication.

Look here:

https://curl.haxx.se/libcurl/c/CURLOPT_PROXYAUTH.html

And here:

https://curl.haxx.se/libcurl/c/CURLOPT_HTTPAUTH.html

Bernd

Re: ReceiveHTTPToMemory() with Proxy support

Posted: Mon Jul 18, 2016 9:03 am
by djes
Infratec is right. There's several way of authenticating, all implemented by curl easily (it's way this lib is great). But you have to handle them yourself. Maybe you could do a little select box to choose and test each of them. For example, code for CURLAUTH_BASIC is

Code: Select all

curl_easy_setopt(curl, #CURLOPT_PROXYAUTH, #CURLAUTH_BASIC)

Re: ReceiveHTTPToMemory() with Proxy support

Posted: Tue Jul 19, 2016 8:33 am
by thyphoon
Hello ! Thanks to infratec for this great code !

I find a hook to replace orginal ReceiveHTTPMemory by your curl wrapper and the Djes ReceiveHTTPToMemory procedure to support Proxy
Know you just add this code with a include and use at the start

Code: Select all

libcurl::InitProxy(ProxyURL$ ,ProxyPort$,ProxyUser$,ProxyPass$)
to setup proxy
and after you can use normaly ReceiveHTTPMemory(URL$)

Code: Select all


DeclareModule libcurl
  Declare InitProxy( ProxyURL.s,ProxyPort.s,ProxyUser.s,ProxyPass.s)
  Declare.i ReceiveHTTPToMemory(URL$)
EndDeclareModule  

Module libcurl
  
  IncludeFile "libcurl.pbi" ; https://github.com/deseven/pbsamples/tree/master/crossplatform/libcurl
  
  Structure http_curl_ext
    ProxyURL.s
    ProxyPort.s
    ProxyUser.s
    ProxyPass.s
  EndStructure
  
  Global http.http_curl_ext
  
  Procedure InitProxy( ProxyURL.s,ProxyPort.s,ProxyUser.s,ProxyPass.s)
    http\ProxyURL=ProxyURL
    http\ProxyPort=ProxyPort
    http\ProxyUser=ProxyUser
    http\ProxyPass=ProxyPass
  EndProcedure
  
  Global *ReceiveHTTPToMemoryBuffer, ReceiveHTTPToMemoryBufferPtr.i
  
  
ProcedureC ReceiveHTTPWriteToMemoryFunction(*ptr, Size.i, NMemB.i, *Stream)
  
  Protected SizeProper.i  = Size & 255
  Protected NMemBProper.i = NMemB
  
  If *ReceiveHTTPToMemoryBuffer = 0
    *ReceiveHTTPToMemoryBuffer = AllocateMemory(SizeProper * NMemBProper)
    If *ReceiveHTTPToMemoryBuffer = 0
      Debug "Problem allocating memory"
      End
    EndIf
  Else
    *ReceiveHTTPToMemoryBuffer = ReAllocateMemory(*ReceiveHTTPToMemoryBuffer, MemorySize(*ReceiveHTTPToMemoryBuffer) + SizeProper * NMemBProper)
    If *ReceiveHTTPToMemoryBuffer = 0
      Debug "Problem reallocating memory"
      End
    EndIf  
  EndIf
  
  CopyMemory(*ptr, *ReceiveHTTPToMemoryBuffer + ReceiveHTTPToMemoryBufferPtr, SizeProper * NMemBProper)
  ReceiveHTTPToMemoryBufferPtr + SizeProper * NMemBProper
  
  ProcedureReturn SizeProper * NMemBProper
  
EndProcedure



Procedure.i ReceiveHTTPToMemory(URL$)
  ProxyURL$=http\ProxyURL
  ProxyPort$=http\ProxyPort
  ProxyUser$=http\ProxyUser
  ProxyPassword$=http\ProxyPass
  
  Protected *Buffer, curl.i, Timeout.i, res.i
  
  If Len(URL$)
    
    curl  = curl_easy_init()
    If curl
      
      Timeout = 3
      
      curl_easy_setopt(curl, #CURLOPT_URL, str2curl(URL$))
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
      curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
      curl_easy_setopt(curl, #CURLOPT_HEADER, 0) ;No headers, please :)
      curl_easy_setopt(curl, #CURLOPT_TIMEOUT, Timeout)
      
      If Len(ProxyURL$)
        ;curl_easy_setopt(curl, #CURLOPT_HTTPPROXYTUNNEL, #True)
        If Len(ProxyPort$)
          ProxyURL$ + ":" + ProxyPort$
        EndIf
        ;Debug ProxyURL$
        curl_easy_setopt(curl, #CURLOPT_PROXY, str2curl(ProxyURL$))
        If Len(ProxyUser$)
          If Len(ProxyPassword$)
            ProxyUser$ + ":" + ProxyPassword$
          EndIf
          ;Debug ProxyUser$
          curl_easy_setopt(curl, #CURLOPT_PROXYUSERPWD, str2curl(ProxyUser$))
        EndIf
      EndIf
      
      curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @ReceiveHTTPWriteToMemoryFunction())
      res = curl_easy_perform(curl)
      
      If res = #CURLE_OK
        *Buffer = AllocateMemory(ReceiveHTTPToMemoryBufferPtr)
        If *Buffer
          CopyMemory(*ReceiveHTTPToMemoryBuffer, *Buffer, ReceiveHTTPToMemoryBufferPtr)
          FreeMemory(*ReceiveHTTPToMemoryBuffer)
          *ReceiveHTTPToMemoryBuffer = #Null
          ReceiveHTTPToMemoryBufferPtr = 0
        Else
          Debug "Problem allocating buffer"         
        EndIf        
        curl_easy_cleanup(curl)
      Else
        Debug "CURL NOT OK"
      EndIf
      
    Else
      Debug "Can't Init CURL"
    EndIf
    
  EndIf
  
  ProcedureReturn *Buffer
  
  
EndProcedure
  
EndModule


;-HOOK to remplace original ReceiveHTTPMemory
Macro ReceiveHTTPMemory(URL)
libcurl::ReceiveHTTPToMemory(URL):MessageRequester("coucou","coucou")
EndMacro  

CompilerIf #PB_Compiler_IsMainFile
 
  InitNetwork()
  ProxyURL$  = InputRequester("ProxyServer", "Do you use a Proxy Server? Then enter the full url:", "")
  ProxyPort$ = InputRequester("ProxyPort"  , "Do you use a specific port? Then enter it", "")
  ProxyUser$ = InputRequester("ProxyUser"  , "Do you use a user name? Then enter it", "")
  ProxyPass$ = InputRequester("ProxyPass"  , "Do you use a password? Then enter it:", "")
  libcurl::InitProxy(ProxyURL$ ,ProxyPort$,ProxyUser$,ProxyPass$)

  *Buffer = ReceiveHTTPMemory("http://www.purebasic.fr/english/index.php")
  If *Buffer
    ShowMemoryViewer(*Buffer, MemorySize(*Buffer))
    FreeMemory(*Buffer)
  Else
    MessageRequester("Erreur", "Problem while loading", #PB_MessageRequester_Ok )
  EndIf
  
CompilerEndIf

Re: ReceiveHTTPToMemory() with Proxy support

Posted: Tue Jun 20, 2017 9:20 pm
by novablue
How do i make this work with socks5 proxy?
All times are UTC+01:00
Page 1 of 1

Powered by phpBB® Forum Software © phpBB Limited