More WinHTTP

RichAlgeni · Post by **RichAlgeni** » Sat Aug 08, 2015 10:40 pm

With thanks to @bgeraghty, I've been playing with WinHTTP. I'm having a strange issue where the header is returned Unicode, and the body in ASCII. It doesn't seem to matter if I compile in ASCII Or Unicode, though the example is in ASCII. You'll see that I had to PeekS the header in Unicode, (lines 141-142 in the include) then PokeS it back in ASCII to get data in the file to show correctly. I did try different WriteStringFormats, none seemed to help. The issue is probably in the header sent to the server, but I need to stop playing for now, and get some other work done.

WinHTTP.pbi

Code: Select all

Import "D:\dev\PureBasic\PureBasic\utilities\Winhttp.lib\winhttp64.lib"
    WinHttpOpen(pwszUserAgent.p-unicode, dwAccessType.i, *pwszProxyName, *pwszProxyBypass, dwFlags.i)
    WinHttpConnect(hSession, pswzServerName.p-unicode, nServerPort.i, dwReserved.i)
    WinHttpSetOption(hInternet, dwOption.i, *lpBuffer, dwBufferLength.i)
    WinHttpSetCredentials(hInternet, AuthTargets.i, AuthScheme.i, pwszUserName.p-unicode, pwszPassword.p-unicode, *pAuthParams)
    WinHttpOpenRequest(hConnect, pwszVerb.p-unicode, pwszObjectName.p-unicode, *pwszVersion, *pwszReferrer, *ppwszAcceptTypes, dwFlags.i)
    WinHttpSendRequest(hRequest, pwszHeaders.p-unicode, dwHeadersLength.i, *lpOptional, dwOptionalLength.i, dwTotalLength.i, dwContext.i)
    WinHttpReceiveResponse(hRequest, *lpReserved)
    WinHttpAddRequestHeaders(hRequest, pwszHeaders.p-unicode, dwHeadersLength.i, dwModifiers.i)
    WinHttpQueryHeaders(hRequest, dwInfoLevel.i, *pwszName, *lpBuffer, *lpdwBufferLength, *lpdwIndex)
    WinHttpQueryDataAvailable(hRequest, *lpdwNumberOfBytesAvailable)
    WinHttpReadData(hRequest, *lpBuffer, dwNumberOfBytesToRead.i, *lpdwNumberOfBytesRead)
    WinHttpCrackUrl(pwszUrl.p-unicode, dwUrlLength.i, dwFlags.i, *lpUrlComponents)
    WinHttpCloseHandle(hInternet)
EndImport

; set constants

#INTERNET_DEFAULT_HTTP_PORT                             = 80
#INTERNET_DEFAULT_HTTPS_PORT                            = 443
#WINHTTP_NO_PROXY_NAME                                  = 0
#WINHTTP_NO_PROXY_BYPASS                                = 0
#WINHTTP_NO_REFERER                                     = 0
#WINHTTP_NO_HEADER_INDEX                                = 0
#WINHTTP_DEFAULT_ACCEPT_TYPES                           = 0
#WINHTTP_ACCESS_TYPE_DEFAULT_PROXY                      = 0
#WINHTTP_HEADER_NAME_BY_INDEX                           = 0
#WINHTTP_AUTH_TARGET_SERVER                             = 0
#WINHTTP_AUTH_TARGET_PROXY                              = 1
#WINHTTP_AUTH_SCHEME_BASIC                              = 1
#WINHTTP_AUTH_SCHEME_NTLM                               = 2
#WINHTTP_AUTH_SCHEME_PASSPORT                           = 4
#WINHTTP_AUTH_SCHEME_DIGEST                             = 8
#WINHTTP_AUTH_SCHEME_NEGOTIATE                          = 16
#WINHTTP_OPTION_REDIRECT_POLICY                         = 88
#WINHTTP_OPTION_REDIRECT_POLICY_NEVER                   = 0
#WINHTTP_OPTION_REDIRECT_POLICY_DISALLOW_HTTPS_TO_HTTP  = 1
#WINHTTP_OPTION_REDIRECT_POLICY_ALWAYS                  = 2
#WINHTTP_QUERY_STATUS_CODE                              = 19
#WINHTTP_QUERY_RAW_HEADERS_CRLF                         = 22
#WINHTTP_QUERY_CONTENT_ENCODING                         = 29
#WINHTTP_QUERY_LOCATION                                 = 33
#WINHTTP_QUERY_FLAG_NUMBER                              = $20000000
#WINHTTP_OPTION_USERNAME                                = $1000
#WINHTTP_OPTION_PASSWORD                                = $1001
#WINHTTP_FLAG_REFRESH                                   = $00000100
#WINHTTP_FLAG_SECURE                                    = $00800000
#WINHTTP_ADDREQ_FLAG_ADD                                = $20000000


Prototype ReceiveHTTPStart(CallbackID, hRequest)
Prototype ReceiveHTTPProgress(CallbackID, lBytesReceived, lSize, lElapsedTime)
Prototype ReceiveHTTPEnd(CallbackID, lRetVal, lBytesReceived, lSize, lElapsedTime)

; If dataToReturn = 1, only the header will be returned, 2 = only the body returned, 1 + 2 (3) will return both the header and body

Procedure ReceiveHTTPMemory(URL$, RequestType$ = "GET", dataToReturn = #False, Username$ = "", Password$ = "", HeaderData$ = "", OptionalData$ = "", UserAgent$ = "WinHTTP - PureBasic", CallbackID = 0, CallbackStart.ReceiveHTTPStart = 0, CallbackProgress.ReceiveHTTPProgress = 0, CallbackEnd.ReceiveHTTPEnd = 0)
    Protected lpUrlComponents.URL_COMPONENTS\dwStructSize = SizeOf(URL_COMPONENTS)
    Protected lStatusCode.i, lContentLen.i, lRedirectPolicy.i = #WINHTTP_OPTION_REDIRECT_POLICY_ALWAYS, lLongSize.i = SizeOf(Long)
    Protected hInternet, hConnect, hRequest, lRetVal, lBytesRead, lReadUntilNow, lBufSize, lStartTime, lResult
    Protected lPort, lFlags, sDomain$, sPath$, sQuery$, *OptionalBuffer, OptionalLength, *MemoryBuffer, MemoryLength
    Protected Result$
    Static hSession

    lStartTime = ElapsedMilliseconds()
    lpUrlComponents\dwSchemeLength    = -1
    lpUrlComponents\dwHostNameLength  = -1
    lpUrlComponents\dwUrlPathLength   = -1
    lpUrlComponents\dwExtraInfoLength = -1

    If WinHttpCrackUrl(URLEncoder(URL$), #Null, #Null, @lpUrlComponents)
        If lpUrlComponents\nScheme = #INTERNET_SCHEME_HTTP
            lPort  = #INTERNET_DEFAULT_HTTP_PORT
            lFlags = #WINHTTP_FLAG_REFRESH
        Else
            lPort  = #INTERNET_DEFAULT_HTTPS_PORT
            lFlags = #WINHTTP_FLAG_REFRESH | #WINHTTP_FLAG_SECURE
        EndIf

        If lpUrlComponents\lpszHostName And lpUrlComponents\dwHostNameLength
            sDomain$ = PeekS(lpUrlComponents\lpszHostName, lpUrlComponents\dwHostNameLength, #PB_Unicode)
        EndIf
        If lpUrlComponents\lpszUrlPath And lpUrlComponents\dwUrlPathLength
            sPath$ = PeekS(lpUrlComponents\lpszUrlPath, lpUrlComponents\dwUrlPathLength, #PB_Unicode)
        EndIf
        If lpUrlComponents\lpszExtraInfo And lpUrlComponents\dwExtraInfoLength
            sQuery$ = PeekS(lpUrlComponents\lpszExtraInfo, lpUrlComponents\dwExtraInfoLength, #PB_Unicode)
        EndIf

        If sDomain$ And sPath$
            If Not hSession
                hSession = WinHttpOpen(UserAgent$, #WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, #WINHTTP_NO_PROXY_NAME, #WINHTTP_NO_PROXY_BYPASS, 0)
            EndIf
            If hSession
                hInternet = WinHttpConnect(hSession, sDomain$, lPort, #Null)
                If hInternet
                    hRequest = WinHttpOpenRequest(hInternet, RequestType$, sPath$+sQuery$, #Null, #WINHTTP_NO_REFERER, #WINHTTP_DEFAULT_ACCEPT_TYPES, lFlags)
                    If hRequest
                        If StringByteLength(OptionalData$, #PB_UTF8)
                            *OptionalBuffer = AllocateMemory(StringByteLength(OptionalData$, #PB_UTF8)+1)
                        EndIf
                        If *OptionalBuffer
                            OptionalLength = MemorySize(*OptionalBuffer)
                            PokeS(*OptionalBuffer, OptionalData$, OptionalLength, #PB_UTF8)
                            OptionalLength - 1
                        EndIf
                        If lpUrlComponents\nScheme = #INTERNET_SCHEME_HTTP
                            WinHttpSetOption(hRequest, #WINHTTP_OPTION_REDIRECT_POLICY, @lRedirectPolicy, SizeOf(Long))
                        EndIf
                        If Len(Username$)
                            WinHttpSetCredentials(hRequest, #WINHTTP_AUTH_TARGET_SERVER, #WINHTTP_AUTH_SCHEME_BASIC, Username$, Password$, #Null)
                        EndIf

                        WinHttpAddRequestHeaders(hRequest, "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"+#CRLF$, -1, #WINHTTP_ADDREQ_FLAG_ADD)
                        WinHttpAddRequestHeaders(hRequest, "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7"+#CRLF$, -1, #WINHTTP_ADDREQ_FLAG_ADD)
                        WinHttpAddRequestHeaders(hRequest, "Accept-Language: en-us,en-gb;q=0.9,en;q=0.8,*;q=0.7"+#CRLF$, -1, #WINHTTP_ADDREQ_FLAG_ADD)

                        If RequestType$ = "POST"
                            WinHttpAddRequestHeaders(hRequest, "Content-Type: application/x-www-form-urlencoded"+#CRLF$, -1, #WINHTTP_ADDREQ_FLAG_ADD)
                        EndIf
                        If CallbackStart
                            CallbackStart(CallbackID, hRequest)
                        EndIf
                        If WinHttpSendRequest(hRequest, HeaderData$, Len(HeaderData$), *OptionalBuffer, OptionalLength, OptionalLength, CallbackID)
                            If WinHttpReceiveResponse(hRequest, #Null)
                                If WinHttpQueryHeaders(hRequest, #WINHTTP_QUERY_FLAG_NUMBER | #WINHTTP_QUERY_STATUS_CODE, #WINHTTP_HEADER_NAME_BY_INDEX, @lStatusCode, @lLongSize, #WINHTTP_NO_HEADER_INDEX)
                                    If lStatusCode  = 200
                                        lResult     = WinHttpQueryDataAvailable(hRequest, @lContentLen)
                                    Else
                                        lResult     = #True
                                        lContentLen = 0
                                    EndIf
                                    If lResult
                                        MemoryLength  = 65000
                                        *MemoryBuffer = AllocateMemory(MemoryLength + 1)

                                        If dataToReturn = 1 Or dataToReturn = 3
                                            lReadUntilNow = 9999
                                            WinHttpQueryHeaders(hRequest, #WINHTTP_QUERY_RAW_HEADERS_CRLF, #WINHTTP_HEADER_NAME_BY_INDEX, *MemoryBuffer, @lReadUntilNow, #WINHTTP_NO_HEADER_INDEX)

Result$ = PeekS(*MemoryBuffer, lReadUntilNow, #PB_Unicode)
lReadUntilNow = PokeS(*MemoryBuffer, Result$, lReadUntilNow, #PB_Ascii)

                                            If dataToReturn = 1
                                                lRetVal = ReAllocateMemory(*MemoryBuffer, lReadUntilNow)
                                            EndIf
                                        EndIf

                                        If dataToReturn = 2 Or dataToReturn = 3
                                            If lContentLen
                                                Repeat
                                                    If  MemoryLength  - lReadUntilNow <= lContentLen
                                                        MemoryLength  = MemoryLength + lContentLen + 1
                                                        *MemoryBuffer = ReAllocateMemory(*MemoryBuffer, MemoryLength)
                                                    EndIf
                                                    If WinHttpReadData(hRequest, *MemoryBuffer + lReadUntilNow, lContentLen, @lBytesRead)
                                                        If lBytesRead
                                                            lReadUntilNow + lBytesRead
                                                        Else
                                                            Break
                                                        EndIf
                                                        If CallbackProgress
                                                            CallbackProgress(CallbackID, lReadUntilNow, lContentLen, (ElapsedMilliseconds() - lStartTime) / 1000)
                                                        EndIf
                                                    Else
                                                        Break
                                                    EndIf
                                                    If Not WinHttpQueryDataAvailable(hRequest, @lContentLen)
                                                        Break
                                                    EndIf
                                                ForEver
                                                If lReadUntilNow >= lContentLen
                                                    lRetVal = ReAllocateMemory(*MemoryBuffer, lReadUntilNow)
                                                EndIf
                                            EndIf
                                        EndIf
                                    EndIf
                                EndIf
                            EndIf
                        EndIf
                        If *OptionalBuffer
                            FreeMemory(*OptionalBuffer)
                        EndIf
                        If CallbackEnd
                            CallbackEnd(CallbackID, lRetVal, lReadUntilNow, lContentLen, (ElapsedMilliseconds() - lStartTime) / 1000)
                        EndIf
                    EndIf
                EndIf
            EndIf
        Else
            errorLine + #CRLF$ + "Both 'sDomain$' And 'sPath$' are null"
        EndIf
    Else
        errorLine + #CRLF$ + "Unable to 'crack' URL: " + URLEncoder(URL$)
    EndIf

    If hRequest
        WinHttpCloseHandle(hRequest)
    EndIf
    If hInternet
        WinHttpCloseHandle(hInternet)
    EndIf

    ProcedureReturn lRetVal
EndProcedure

Procedure.s ReceiveHTTPString(URL$, RequestType$ = "GET", dataToReturn = 1, Username$ = "", Password$ = "", HeaderData$ = "", OptionalData$ = "", UserAgent$ = "WinHTTP - PureBasic", CallbackID = 0, CallbackStart.ReceiveHTTPStart = 0, CallbackProgress.ReceiveHTTPProgress = 0, CallbackEnd.ReceiveHTTPEnd = 0)
    Protected Result$ = ""
    Protected *MemoryBuffer
    *MemoryBuffer = ReceiveHTTPMemory(URL$, RequestType$, dataToReturn, Username$, Password$, HeaderData$, OptionalData$, UserAgent$, CallbackID, CallbackStart.ReceiveHTTPStart, CallbackProgress.ReceiveHTTPProgress, CallbackEnd.ReceiveHTTPEnd)
    If *MemoryBuffer And dataToReturn
        Result$ = PeekS(*MemoryBuffer, MemorySize(*MemoryBuffer), #PB_UTF8)
        FreeMemory(*MemoryBuffer)
    EndIf
    ProcedureReturn Result$
EndProcedure

Procedure ReceiveHTTPFileEx(URL$, Filename$, RequestType$ = "GET", dataToReturn = 1, Username$ = "", Password$ = "", HeaderData$ = "", OptionalData$ = "", UserAgent$ = "WinHTTP - PureBasic", CallbackID = 0, CallbackStart.ReceiveHTTPStart = 0, CallbackProgress.ReceiveHTTPProgress = 0, CallbackEnd.ReceiveHTTPEnd = 0)
    Protected fileNumber.i
    Protected *MemoryBuffer
    Protected result.i
    *MemoryBuffer = ReceiveHTTPMemory(URL$, RequestType$, dataToReturn, Username$, Password$, HeaderData$, OptionalData$, UserAgent$, CallbackID, CallbackStart.ReceiveHTTPStart, CallbackProgress.ReceiveHTTPProgress, CallbackEnd.ReceiveHTTPEnd)
    If *MemoryBuffer
        fileNumber = CreateFile(#PB_Any, Filename$)
        If fileNumber
            WriteData(fileNumber, *MemoryBuffer, MemorySize(*MemoryBuffer))
            CloseFile(fileNumber)
            result = #True
        Else
            result = #False
        EndIf
        FreeMemory(*MemoryBuffer)
    Else
        errorLine + #CRLF$ + "Nothing found with " + RequestType$ + " request to " + URL$
        result    = #False
    EndIf
    ProcedureReturn result
EndProcedure

WinHTTP_Test.pb

Code: Select all

EnableExplicit

Define thisURL.s   = "https://twitter.com/PureBasic_DVP"
Global errorLine.s
Define foundOk.i
Define displayResult.s
Define dataToReturn.i = 3

IncludeFile "D:\dev\PureBasic\utilities\Winhttp.lib\WinHTTP.pbi"

foundOk = ReceiveHTTPFileEx(thisURL, "D:\dev\PureBasic\utilities\Winhttp.lib\testit.txt", "GET", dataToReturn)

If foundOk
    displayResult = "Page received from " + thisURL + " ok!"
Else
    displayResult = "Nothing found with request to " + thisURL + errorLine
EndIf

MessageRequester("Winhttp_test", displayResult)
; IDE Options = PureBasic 5.24 LTS (Windows - x64)
; CursorPosition = 2
; EnableXP
; DisableDebugger
; HideErrorLog
; CurrentDirectory = D:\dev\PureBasic\PureBasic\
; CompileSourceDirectory

I've also modified the code a bit to return the header, body, or both.

If I compile in Unicode mode, the file results in the header being readable (written in unicode), but the body shows up looking like it's in Chinese!

RichAlgeni · Post by **RichAlgeni** » Sat Aug 08, 2015 10:47 pm

I should add, the image is of the file created when the program is compiled in Unicode, and lines 141-142 are commented out.

IdeasVacuum · Post by **IdeasVacuum** » Sun Aug 09, 2015 12:43 am

...ensure that your source code page is UTF8. It should not matter, but it does.

RichAlgeni · Post by **RichAlgeni** » Wed Aug 12, 2015 1:14 am

IdeasVacuum wrote:...ensure that your source code page is UTF8. It should not matter, but it does.

Haven't had a chance to try this yet, but I appreciate your feedback as always!

PureBasic Forums - English

More WinHTTP

More WinHTTP

Re: More WinHTTP

Re: More WinHTTP

Re: More WinHTTP