How to grab a https page with cURL/libcURL

[Lord]

How do I grab a page that is shown with Firefox
but not with WebGadget() or IE?
For example this link:
https://familysearch.org/records/collec ... /waypoints

Code: Select all

IncludeFile "libcurl-res.pb"
IncludeFile "libcurl-inc.pb"
; Includes found here:
; http://www.purebasic.fr/english/viewtopic.php?f=12&t=49758


ProcedureC  LibCurlWriteFunction(*ptr, Size.i, NMemB, *Stream)
  Protected SizeProper.i  = Size & 255
  Protected NMemBProper.i = NMemB
  Protected MyDataS.s
  Shared ReceivedData.s
  MyDataS = PeekS(*ptr, -1, #PB_UTF8):ShowMemoryViewer(*ptr, MemorySize(*ptr))
  ReceivedData + MyDataS
  ProcedureReturn  SizeProper * NMemBProper
EndProcedure
Procedure.s LibCurlGetData()
  Shared ReceivedData.s
  Protected ReturnData.s
  ReturnData.s = ReceivedData.s
  ReceivedData.s = ""
  ProcedureReturn ReturnData.s
EndProcedure

URL.s="https://familysearch.org/records/collection/1951790/waypoints"

curl  = curl_easy_init()
If curl
  Debug curl_easy_setopt(curl, #CURLOPT_URL, @URL)
  Debug curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
  Debug curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
  Debug curl_easy_setopt(curl, #CURLOPT_HEADER, @"")
  Debug curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @RW_LibCurl_WriteFunction())
  res = curl_easy_perform(curl)
  Debug res
  Debug "----"
  Debug  LibCurlGetData()
  Debug "----"
  Debug curl_easy_cleanup(curl)
EndIf

The code results in a "404 Not found":

0
0
0
0
0
0
----
HTTP/1.1 404 Not Found
Server: Apache-Coyote/1.1
Transfer-Encoding: chunked
Date: Sat, 22 Mar 2014 19:10:06 GMT
Connection: close
Access-Control-Allow-Methods: OPTIONS, HEAD, GET, PUT, POST, DELETE
Access-Control-Allow-Headers: accept, accept-charset, accept-encoding, accept-language, accept-datetime, authorization, connection, content-length, content-md5, content-type, date, expect, from, host, if-match, if-modified-since, if-none-match, if-range, if-unmodified-since, origin, range, referer, singularityjsheader, te, user-agent, warning, x-reason, x-requested-with, x-fs-feature-tag
Access-Control-Expose-Headers: location, link, warning, x-entity-id, content-location, x-processing-time, retry-after
Access-Control-Allow-Origin: *
Access-Control-Max-Age: 604800


----
0

What ist wrong/missing to show page content?

The page is also not shown with IE.
Browser used: FireFox 26 and IE 11.
I need a Windows only way to get the page content.

Any help appreciated.

[infratec]

Hi,

without curl:

Code: Select all

EnableExplicit


#UserAgent$ = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

; Microsoft constants

#INTERNET_DEFAULT_HTTP_PORT = 80
#INTERNET_DEFAULT_HTTPS_PORT = 443

#INTERNET_OPEN_TYPE_DIRECT = 1
#INTERNET_OPEN_TYPE_PROXY = 3

#INTERNET_SERVICE_HTTP = 3

#INTERNET_FLAG_PRAGMA_NOCACHE =           $00000100
#INTERNET_FLAG_IGNORE_CERT_CN_INVALID =   $00001000
#INTERNET_FLAG_IGNORE_CERT_DATE_INVALID = $00002000
#INTERNET_FLAG_NO_COOKIES =               $00080000
#INTERNET_FLAG_KEEP_CONNECTION =          $00400000
#INTERNET_FLAG_SECURE =                   $00800000
#INTERNET_FLAG_DONT_CACHE =               $04000000


#HTTP_ADDREQ_FLAG_ADD = $20000000
#HTTP_ADDREQ_FLAG_REPLACE = $80000000


#INTERNET_OPTION_SECURITY_FLAGS = 31
#SECURITY_FLAG_IGNORE_UNKNOWN_CA = $00000100


Structure ServerStructure
  UserAgent$
  Secure.i
  OpenHandle.i
  ConnectionHandle.i
  RequestHandle.i
  KeepAlive.i
  Server$
  Path$
  Parameters$
  User$
  Password$
  Base64Auth$
EndStructure


Structure HTTPAnswerSructure
  Status.i
  Content$
EndStructure





Procedure.i OpenConnection(*Info.ServerStructure)
 
  Protected.l timeout
  Protected.i Result, port
 
 
  *Info\OpenHandle = InternetOpen_(*Info\UserAgent$, #INTERNET_OPEN_TYPE_DIRECT, "", "", 0)
 
  If *Info\OpenHandle
   
    timeout = 1000
    InternetSetOption_(*Info\OpenHandle, 2, timeout, 4)
   
    If *Info\Secure
      *Info\KeepAlive = #True
      port = #INTERNET_DEFAULT_HTTPS_PORT
    Else
      port = #INTERNET_DEFAULT_HTTP_PORT
    EndIf
   
    *Info\ConnectionHandle = InternetConnect_(*Info\OpenHandle, *Info\Server$, port, "", "", #INTERNET_SERVICE_HTTP, 0, 0)
    If *Info\ConnectionHandle
      Result = #True
    EndIf
   
  EndIf
 
  ProcedureReturn Result
 
EndProcedure




Procedure.i SendHeader(*Info.ServerStructure, Header$)
 
  Protected.l Flag
  Protected.i Result, Pos
  Protected FirstLine$
 
 
  Pos = FindString(Header$, #CRLF$)
  FirstLine$ = Left(Header$, Pos)
  Header$ = Mid(Header$, Pos + 2)
 
 
  Flag = #INTERNET_FLAG_PRAGMA_NOCACHE
  Flag | #INTERNET_FLAG_NO_COOKIES
  Flag | #INTERNET_FLAG_DONT_CACHE
 
  If *Info\Secure
    Flag | #INTERNET_FLAG_SECURE
    Flag | #INTERNET_FLAG_IGNORE_CERT_CN_INVALID
    Flag | #INTERNET_FLAG_IGNORE_CERT_DATE_INVALID
  EndIf
 
  If *Info\KeepAlive
    Flag | #INTERNET_FLAG_KEEP_CONNECTION
  EndIf
 
  *Info\RequestHandle = HttpOpenRequest_(*Info\ConnectionHandle, StringField(FirstLine$, 1, " "), StringField(FirstLine$, 2, " "), "", "", #Null, flag, 0)
  If *Info\RequestHandle
    HttpAddRequestHeaders_(*Info\RequestHandle, Header$, Len(Header$), #HTTP_ADDREQ_FLAG_ADD|#HTTP_ADDREQ_FLAG_REPLACE)
    Result = #True
  EndIf
   
  ProcedureReturn Result
   
EndProcedure




Procedure.i SendBody(*Info.ServerStructure, Body$)
 
  Protected.l dwFlags, dwBuffLen
  Protected.i Result, Ok, SendHandle
 
 
  Repeat
    SendHandle = HttpSendRequest_(*Info\RequestHandle, #Null, 0, @Body$, Len(Body$))
    ;Debug "SendHandle " + Str(SendHandle)
    If SendHandle = 0
      Debug GetLastError_()
      If GetLastError_() = 12045
       
        Debug "cert"
       
        dwBuffLen = SizeOf(dwFlags)
        InternetQueryOption_(*Info\RequestHandle, #INTERNET_OPTION_SECURITY_FLAGS, @dwFlags, @dwBuffLen)
       
        dwFlags | #SECURITY_FLAG_IGNORE_UNKNOWN_CA
        InternetSetOption_(*Info\RequestHandle, #INTERNET_OPTION_SECURITY_FLAGS, @dwFlags, SizeOf(dwFlags))
        Ok + 1
      Else
        Ok = 3
      EndIf
    Else
      Ok = 2
    EndIf
  Until Ok >= 2
 
 
  If OK = 2
    Result = #True
  Else
    Result = #False
  EndIf
 
  If SendHandle
    InternetCloseHandle_(SendHandle)
  EndIf
 
  ProcedureReturn Result
 
EndProcedure




Procedure.i ReceiveAnswer(*Info.ServerStructure, *Answer.HTTPAnswerSructure)
  
  Protected.l Count
  Protected.i Result, Help
  Protected *Buffer
  
  
  *Answer\Status = 0
  *Answer\Content$ = ""
  
  *Buffer = AllocateMemory(1024)
  If *Buffer
    Repeat
      Help = InternetReadFile_(*Info\RequestHandle, *Buffer, 1024, @Count)
      If Help = #True And Count = 0
        
        ; if the file download is complete
        
        Count = 10
        If HttpQueryInfo_(*Info\RequestHandle, #HTTP_QUERY_STATUS_CODE, *Buffer, @Count, #Null)
          *Answer\Status = Val(PeekS(*Buffer, Count))
        EndIf
        
        Result = #True
      Else
        If GetLastError_() = 0
          *Answer\Content$ + PeekS(*Buffer, Count, #PB_UTF8)
        Else
          *Answer\Content$ = ""
          Break
        EndIf
      EndIf
    Until Result
    FreeMemory(*Buffer)
  EndIf
  
  ProcedureReturn Result
  
EndProcedure




Procedure.s HttpReceive(URL$)
 
  Protected.i request_handle
  Protected Header$, Post$
  Protected Info.ServerStructure, Answer.HTTPAnswerSructure

 
  If LCase(GetURLPart(URL$, #PB_URL_Protocol)) = "https"
    Info\Secure = #True
  EndIf
 
  Info\Server$ = GetURLPart(URL$, #PB_URL_Site)
  Info\Path$ = GetURLPart(URL$, #PB_URL_Path)
 
  Info\Parameters$ = GetURLPart(URL$, #PB_URL_Parameters)
  If Len(Info\Parameters$)
    Info\Parameters$ = "?" + Info\Parameters$
  EndIf

  OpenConnection(@Info)
 
 
  If Info\ConnectionHandle
    Post$ = ""
   
    Header$ = "GET /" + Info\Path$ + Info\Parameters$ + #CRLF$
    Header$ + "Host: " + Info\Server$ + #CRLF$
    Header$ + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + #CRLF$
    Header$ + #CRLF$
   
    ;Debug Header$
   
    If SendHeader(@Info, Header$)
      If SendBody(@Info, Post$)
        
        If ReceiveAnswer(@Info, @Answer)
          Debug Answer\Status
          ;Debug Answer\Content$
        Else
          Debug Answer\Status
          Debug Answer\Content$
        EndIf
        
      EndIf
      
      InternetCloseHandle_(Info\RequestHandle)
      
    EndIf
    
    InternetCloseHandle_(Info\ConnectionHandle)
    InternetCloseHandle_(Info\OpenHandle)
    
  EndIf
  
  ProcedureReturn Answer\Content$
  
EndProcedure


Debug HttpReceive("https://familysearch.org/records/collection/1951790/waypoints")
Debug "---------------------------------"
Debug HttpReceive("https://familysearch.org/recapi/sord/waypoint/M6YL-QP6:361426101?cc=2101574")

It was a bit tricky:
The website does not return a content length.
So my first try runs in an error too.
Than I simply download the file without a real content length.
At the moment I use a fixed buffersize of 500000.
The file is 257xxx at the moment.

Bernd

[Lord]

Hi Bernd!

Your code works perfect. Didn't know that https can be done this way.
But now i stumbled over another URL that is shown with FireFox but
is not loaded by your code:

https://familysearch.org/recapi/sord/wa ... cc=2101574

Debug output shows only the Request Handle.
Is there also a way to get that content? What has to be modified?

[infratec]

Hi,

I extended the code above.

Bernd

[Lord]

Hi Bernd!

Your updated code works perfect and suits my needs.
Thanks for sharing this nice piece of code.

[infratec]

Hi,

I updated the listing above to make it more 'consistant' and less resource hungry

Bernd

[Lord]

Hi Bernd!

I will try your update to.
Thanks for your work. Its very helpfull.

[Falko]

Hi Bernd,

How can i make with your Source at User and Password?
I use lib curl for that at phonegap.com to read my app or delete my app.

The phonegap api write it so in curl

http://docs.build.phonegap.com/en_US/3. ... te.md.html

You can scroll down to see the curl-commands.

The follow ist to delete an app, if i use the AppID

DELETE https://build.phonegap.com/api/v1/apps/:id

Delete your application from PhoneGap Build, returning either a
202 (accepted) status, or 404 (if the app cannot be found):

$ curl -u andrew.lunny@nitobi.com -X DELETE https://build.phonegap.com/api/v1/apps/8
{
"success":"app 8 deleted"
}

In libCurl i've coding it follow. But here i must use many dll from lib-curl for that.

Code: Select all

EnableExplicit

XIncludeFile "RW_LibCurl_Res.pb"
XIncludeFile "RW_LibCurl_Inc.pb"

Global Window_0

Global String_0, String_1, String_2, Text_0, Text_1, Text_2, Button_1, Button_0

Declare PhoneGap(AppID.s,Benutzername.s,Passwort.s,Methode.b)


Define.l Event,Quit

Enumeration FormFont
  #Font_Window_0_0
  #Font_Window_0_1
EndEnumeration

LoadFont(#Font_Window_0_0,"@Arial Unicode MS", 10)
LoadFont(#Font_Window_0_1,"@Arial Unicode MS", 12)


Procedure OpenWindow_0(x = 0, y = 0, width = 400, height = 200)
  Window_0 = OpenWindow(#PB_Any, x, y, width, height, "Delete APP from PhoneGap", #PB_Window_SystemMenu | #PB_Window_MinimizeGadget | #PB_Window_MaximizeGadget)
  String_0 = StringGadget(#PB_Any, 130, 10, 90, 30, "")
  SetGadgetFont(String_0, FontID(#Font_Window_0_0))
  String_1 = StringGadget(#PB_Any, 130, 50, 250, 30, "")
  SetGadgetFont(String_1, FontID(#Font_Window_0_0))
  String_2 = StringGadget(#PB_Any, 130, 90, 250, 30, "",#PB_String_Password)
  SetGadgetFont(String_2, FontID(#Font_Window_0_0))
  Text_0 = TextGadget(#PB_Any, 10, 10, 60, 20, "AppID")
  SetGadgetFont(Text_0, FontID(#Font_Window_0_0))
  Text_1 = TextGadget(#PB_Any, 10, 50, 100, 30, "Benutzername")
  SetGadgetFont(Text_1, FontID(#Font_Window_0_0))
  Text_2 = TextGadget(#PB_Any, 10, 90, 90, 30, "Passwort")
  SetGadgetFont(Text_2, FontID(#Font_Window_0_0))
  Button_0 = ButtonGadget(#PB_Any, 40, 140, 130, 40, "Read PhoneGap")
  SetGadgetFont(Button_0, FontID(#Font_Window_0_1))
  Button_1 = ButtonGadget(#PB_Any, 230, 140, 150, 40, "Delete APP")
  SetGadgetFont(Button_1, FontID(#Font_Window_0_1))
EndProcedure

Procedure Window_0_Events(event)
  Select event
    Case #PB_Event_CloseWindow
      ProcedureReturn #False

    Case #PB_Event_Menu
      Select EventMenu()
      EndSelect

    Case #PB_Event_Gadget
      Select EventGadget()
        Case Button_0
          PhoneGap("",GetGadgetText(String_1),GetGadgetText(String_2),0)
        Case Button_1
          PhoneGap(GetGadgetText(String_0),GetGadgetText(String_1),GetGadgetText(String_2),1)
      EndSelect
  EndSelect
  ProcedureReturn #True
EndProcedure

Procedure PhoneGap(AppID.s,Benutzername.s,Passwort.s,Methode.b)

Protected.s URL,url1,UserPwd,Ausgabe
  Protected.l curl,res,Anfang,Ende,Event
  UserPwd=Benutzername+":"+Passwort
  
  URL.s = "https://build.phonegap.com/api/v1/apps/"+AppID
  
  curl = curl_easy_init()
  If curl
    curl_easy_setopt(curl, #CURLOPT_URL, @URL)
    If Methode=1
          curl_easy_setopt(curl, #CURLOPT_CUSTOMREQUEST, @"DELETE"); Falls ihr ..welche App... diese Zeile auskommentieren
    ElseIf Methode=0
    EndIf
    
    curl_easy_setopt(curl, #CURLOPT_USERPWD, @UserPwd)
    curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYPEER, 0)
    curl_easy_setopt(curl, #CURLOPT_SSL_VERIFYHOST, 0)
    curl_easy_setopt(curl, #CURLOPT_HEADER, 0); @"")
    curl_easy_setopt(curl, #CURLOPT_WRITEFUNCTION, @RW_LibCurl_WriteFunction())
    res = curl_easy_perform(curl)
       
    
    Ausgabe=RW_LibCurl_GetData()
    
    Anfang=FindString(Ausgabe, Chr(34)+"id"+Chr(34)+":", 0)+5
    
    If Mid(Ausgabe,Anfang-5,5)=Chr(34)+"id"+Chr(34)+":"
      Ende=FindString(Ausgabe, ",", Anfang)
      SetGadgetText(String_0,Mid(Ausgabe,Anfang,Ende-Anfang))
    Else
      SetGadgetText(String_0,"")
    EndIf
    MessageRequester("Ausgabe", Ausgabe)
    curl_easy_cleanup(curl)
   EndIf
EndProcedure


 OpenWindow_0()

 Repeat
     Event = WindowEvent()
     Window_0_Events(Event)
     If Event = #PB_Event_CloseWindow  ; If the user has pressed on the close button
       Quit = 1
     EndIf
 
   Until Quit = 1


End   ; All the opened windows are closed automatically by PureBasic

Falko

[infratec]

Hi Falko,

not tested, since I don't know from what you are talking,
but it may work

Code: Select all

Procedure.i BuildBase64Auth(*Info.ServerStructure)
  
  Protected Result.i, Auth$, AuthSize.i, Base64AuthSize.i, Base64Auth$
  
  
  Auth$ = *Info\User$ + ":" + *Info\Password$
  AuthSize = Len(Auth$)
  Base64AuthSize = AuthSize * 1.4
  If Base64AuthSize < 64 : Base64AuthSize = 64 : EndIf
  Base64Auth$ = Space(Base64AuthSize)
  If Base64Encoder(@Auth$, AuthSize, @Base64Auth$, Base64AuthSize)
    *Info\Base64Auth$ = Base64Auth$  
    Result = #True
  EndIf
  
  ProcedureReturn Result
  
EndProcedure


Procedure.s PhoneGap(AppID$, Benutzername$, Passwort$, Methode.i=0)
 
  Protected.i Pos
  Protected Header$, Post$, URL$, Result$
  Protected Info.ServerStructure, Answer.HTTPAnswerSructure
  
  
  URL$ = "https://build.phonegap.com/api/v1/apps/" + AppID$
  
  Info\User = Benutzername$
  Info\Password$ = Passwort$
  BuildBase64Auth(@Info)
  
  If LCase(GetURLPart(URL$, #PB_URL_Protocol)) = "https"
    Info\Secure = #True
  EndIf
  
  Info\Server$ = GetURLPart(URL$, #PB_URL_Site)
  Info\Path$ = GetURLPart(URL$, #PB_URL_Path)
  
  Info\Parameters$ = GetURLPart(URL$, #PB_URL_Parameters)
  If Len(Info\Parameters$)
    Info\Parameters$ = "?" + Info\Parameters$
  EndIf
  
  OpenConnection(@Info)
  
  
  If Info\ConnectionHandle
    Post$ = ""
    
    If Methode = 1
      Header$ = "DELETE /" + Info\Path$ + Info\Parameters$ + #CRLF$
    Else
      Header$ = "GET /" + Info\Path$ + Info\Parameters$ + #CRLF$
    EndIf
    Header$ + "Host: " + Info\Server$ + #CRLF$
    Header$ + "Authorization: Basic " + Info\Base64Auth$ + #CRLF$
    Header$ + #CRLF$
   
    ;Debug Header$
   
    If SendHeader(@Info, Header$)
      If SendBody(@Info, Post$)
       
        If ReceiveAnswer(@Info, @Answer)
          ;Debug Answer\Status
          ;Debug Answer\Content$
          
          If Methode = 0
            If Answer\Status >= 200 And Answer\Status < 300
              Pos = FindString(Answer\Content$, #DQUOTE$ + "id" + #DQUOTE$ + ":")
              If Pos
                Pos + 5
                Result$ = Mid(Answer\Content$, Pos, FindString(Answer\Content$, ",", Pos) - Pos)
              EndIf
            Else
              Result$ = Str(Answer\Status)
            EndIf
          Else
            Result$ = Str(Answer\Status)
          EndIf
          
        Else
          Debug Answer\Status
          Debug Answer\Content$
        EndIf
       
      EndIf
     
      InternetCloseHandle_(Info\RequestHandle)
     
    EndIf
   
    InternetCloseHandle_(Info\ConnectionHandle)
    InternetCloseHandle_(Info\OpenHandle)
   
  EndIf
  
  ProcedureReturn Result$
 
EndProcedure

This are only the 'new' parts.
You have to add them in the code above.
Than you can remove the HttpReceive() procedure.

Oh, I forgot:
You have to add

Code: Select all

Base64Auth$

to the ServerStructure

Bernd

[Falko]

Wow, this is a very good code.

Thank you Bernd.
Here my Test - Output to Read my AppID from PhoneGap.com:

200
{"link":"/api/v1/apps","apps":[{"status":{"symbian":"skip","winphone":"complete","android":"complete","ios":null,"blackberry":"skip","webos":"skip"},"repo":null,"link":"/api/v1/apps/840126","description":"Get a picture from a file or camera and display.","title":"Camera2","phonegap_version":"3.1.0","debug":false,"icon":{"link":"/api/v1/apps/840126/icon","filename":"LauncherIcon57.png"},"build_count":2,"package":"com.nsbasic.CameraToServer1","error":{},"private":true,"version":"1.0.0","download":{"android":"/api/v1/apps/840126/android","winphone":"/api/v1/apps/840126/winphone"},"id":840126,"role":"admin","hydrates":false}]}

One Ask,
is this Methode.i for "Delete", i think?

At ... I don't know...

I use NSB/AppStudio to create basic or java code for web and App for Android etc.
PhoneGap is a Server how i can upload this Java or basic and i get a AppID for android or WindowsPhone.
I've have a free account for 1App and i go online to delete this before i can upload the next code.

Sorry my english.

Falko

[infratec]

Hi,

I thought that I did it like your code:

Methode 1 is for delete, Methode 0 is for read.

Both should work.

Bernd

[Falko]

Sorry, I've overlooked this.

Code: Select all

 If Methode = 1
      Header$ = "DELETE /" + Info\Path$ + Info\Parameters$ + #CRLF$
    Else
      Header$ = "GET /" + Info\Path$ + Info\Parameters$ + #CRLF$
    EndIf

Thank you,
Falko

[infratec]

Hi,

I updated my listing above.
Now it returns directly the id or the status code.

Bernd

[Falko]

Hi,

This ist very usefull for me. Thank you .
I've changed for me into Procedure PhoneGape follow Lines

Protected Header$, Post$, URL$,Result$
Info\User$ = Benutzername$

and

Code: Select all

Header$ + "Authorization: Basic " + *Info\Base64Auth$ + #CRLF$

to

Code: Select all

 Header$ + "Authorization: Basic " + Info\Base64Auth$ + #CRLF$

So it runs perfectly on Phonegap.com - Free-Acount.

Regards,
Falko

[infratec]

Hi,

I corrected the stuff.
Than it's easier for other possible users

It was programmed 'blind', from an other code and without possibility to test.
So I didn't ran the code and ovreseen this small bugs.

Bernd