Adding "ignore self-signed certificate" option (#68)

* added insecure option

added the insecure option to the type Request

* added the insecure option to newRequest function

added the insecure option to the newRequest function, by default it is false (meaning secure)

* added insecure options for curl

is the option for insecure is set, then the insecure options for curl or set. That being to disable peer, and host verification.

* added flags and error to windef

added flags for setting insecure options, and an error for secure failure.

* added another error to windefs

added the error to check for a resend request., because sometimes Negotiate authorization handshakes may return this error

* added WinHttpSetOption

added the WinHttpSetOption function from the dynamic library winhttp. This will allow us the set security options on the request.

* added another flag, and another error

Added the WINHTTP_OPTION_SECURITY_FLAGS for setting security options, and the ERROR_INTERNET_INVALID_CA error, for catching another security error

* added insecure option for windows

This implementation allows for insecure connection if specified. This required adding some more definitions, and procedures to windefs.nim

* added setAllowsAnyHTTPSCertificate

adding the setAllowsAnyHTTPSCerfiticate function.

* Update test.nim

* added python test server

This python server utilizes self signed certificates for HTTPS communication for testing the insecure ssl PR draft.

* added self signed certs

added self signed certificates utilizing openSSL.

* removed testing variables

I've forgot to remove some variables for testing when making the test server.

* added requested updates

Thank you @guzba for giving great ideas to add to this PR, I've added a smaller test server, updated the variable name, and thanks again @guzba for showing me how to use https without a loop for windows.

* fixing commit pull messup

messed up on commit pull, this might fix confilcts

* again messing up the commit

Been a while working on this repository, hopefully this fixes an conflicts.

* added requested fixes... again

Alright somethings I'll have to fix in my script, but I went back and edited them manually. Please let me know if I've forgotten anything.

* added function parameters

I've added the function parameters requested, must've forgotten about them during manual editing.

* removed exception

Removed the exception at the line specified to remove at.

src/puppy.nim

@@ -36,15 +36,17 @@ proc newRequest*(
   url: string,
   verb = "get",
   headers = newSeq[Header](),
-  timeout: float32 = 60
+  timeout: float32 = 60,
+  allowAnyHttpsCertificate: bool = false,
 ): Request =
   ## Allocates a new request object with defaults.
   result = Request()
   result.url = parseUrl(url)
   result.verb = verb
   result.headers = headers
   result.timeout = timeout
-
+  result.allowAnyHttpsCertificate = allowAnyHttpsCertificate
+
 proc fetch*(url: string, headers = newSeq[Header]()): string =
   let
     req = newRequest(url, "get", headers)

src/puppy/common.nim

@@ -15,12 +15,13 @@ type
     timeout*: float32
     verb*: string
     body*: string
+    when defined(puppyLibcurl) or (defined(windows) or not defined(macosx)):
+      allowAnyHttpsCertificate*: bool
 
   Response* = ref object
     headers*: seq[Header]
     code*: int
     body*: string
-
   PuppyError* = object of IOError ## Raised if an operation fails.
 
 proc `[]`*(headers: seq[Header], key: string): string =
@@ -40,4 +41,4 @@ proc `[]=`*(headers: var seq[Header], key, value: string) =
   headers.add(Header(key: key, value: value))
 
 proc `$`*(req: Request): string =
-  req.verb.toUpperAscii & " " & $req.url
+  req.verb.toUpperAscii & " " & $req.url

src/puppy/platforms/linux/platform.nim

@@ -16,29 +16,27 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
     count: int,
     outstream: pointer
   ): int {.cdecl.} =
-    if size != 1:
-      raise newException(PuppyError, "Unexpected curl write callback size")
     let
       outbuf = cast[ptr StringWrap](outstream)
       i = outbuf.str.len
     outbuf.str.setLen(outbuf.str.len + count)
     copyMem(outbuf.str[i].addr, buffer, count)
     result = size * count
-
+   
   {.pop.}
-
+  
   var strings: seq[string]
   strings.add $req.url
   strings.add req.verb.toUpperAscii()
   for header in req.headers:
     strings.add header.key & ": " & header.value
-
+  
   let curl = easy_init()
-
+  
   discard curl.easy_setopt(OPT_URL, strings[0].cstring)
   discard curl.easy_setopt(OPT_CUSTOMREQUEST, strings[1].cstring)
   discard curl.easy_setopt(OPT_TIMEOUT, req.timeout.int)
-
+  
   # Create the Pslist for passing headers to curl manually. This is to
   # avoid needing to call slist_free_all which creates problems
   var slists: seq[Slist]
@@ -54,32 +52,36 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
       while tail.next != nil:
         tail = tail.next
       tail.next = slists[i].addr
-
+  
   discard curl.easy_setopt(OPT_HTTPHEADER, headerList)
-
+  
   if req.verb.toUpperAscii() == "POST" or req.body.len > 0:
     discard curl.easy_setopt(OPT_POSTFIELDSIZE, req.body.len)
     discard curl.easy_setopt(OPT_POSTFIELDS, req.body.cstring)
-
+  
   # Setup writers.
   var headerWrap, bodyWrap: StringWrap
   discard curl.easy_setopt(OPT_WRITEDATA, bodyWrap.addr)
   discard curl.easy_setopt(OPT_WRITEFUNCTION, curlWriteFn)
   discard curl.easy_setopt(OPT_HEADERDATA, headerWrap.addr)
   discard curl.easy_setopt(OPT_HEADERFUNCTION, curlWriteFn)
-
+  
   # On windows look for cacert.pem.
   when defined(windows):
     discard curl.easy_setopt(OPT_CAINFO, "cacert.pem".cstring)
   # Follow redirects by default.
   discard curl.easy_setopt(OPT_FOLLOWLOCATION, 1)
 
+  if req.allowAnyHttpsCertificate:
+    discard curl.easy_setopt(OPT_SSL_VERIFYPEER, 0)
+    discard curl.easy_setopt(OPT_SSL_VERIFYHOST, 0)
+
   let
     ret = curl.easy_perform()
     headerData = headerWrap.str
-
+  
   curl.easy_cleanup()
-
+  
   if ret == E_OK:
     var httpCode: uint32
     discard curl.easy_getinfo(INFO_RESPONSE_CODE, httpCode.addr)
@@ -95,4 +97,4 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
       except ZippyError as e:
         raise newException(PuppyError, "Error uncompressing response", e)
   else:
-    raise newException(PuppyError, $easy_strerror(ret))
+    raise newException(PuppyError, $easy_strerror(ret))

src/puppy/platforms/macos/platform.nim

@@ -17,7 +17,7 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
 
     if req.body.len > 0:
       request.setHTTPBody(NSData.dataWithBytes(req.body[0].addr, req.body.len))
-
+      
     var
       response: NSHTTPURLResponse
       error: NSError

src/puppy/platforms/win32/platform.nim

@@ -114,9 +114,44 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
       req.body.len.DWORD,
       0
     ) == 0:
-      raise newException(
-        PuppyError, "WinHttpSendRequest error: " & $GetLastError()
-      )
+
+      let error = GetLastError()
+      if error in {ERROR_WINHTTP_SECURE_FAILURE, ERROR_INTERNET_INVALID_CA} and
+        req.allowAnyHttpsCertificate:
+        # If this is a certificate error but we should allow any HTTPS cert,
+        # we need to set some options and retry sending the request.
+        # https://stackoverflow.com/questions/19338395/how-do-you-use-winhttp-to-do-ssl-with-a-self-signed-cert
+        var flags: DWORD =
+          SECURITY_FLAG_IGNORE_UNKNOWN_CA or
+          SECURITY_FLAG_IGNORE_CERT_WRONG_USAGE or
+          SECURITY_FLAG_IGNORE_CERT_CN_INVALID or
+          SECURITY_FLAG_IGNORE_CERT_DATE_INVALID
+        if WinHttpSetOption(
+          hRequest,
+          WINHTTP_OPTION_SECURITY_FLAGS,
+          flags.addr,
+          sizeof(flags).DWORD
+        ) == 0:
+          raise newException(
+            PuppyError, "WinHttpSetOption error: " & $GetLastError()
+          )
+
+        if WinHttpSendRequest(
+          hRequest,
+          nil,
+          0,
+          req.body.cstring,
+          req.body.len.DWORD,
+          req.body.len.DWORD,
+          0
+        ) == 0:
+          raise newException(
+            PuppyError, "WinHttpSendRequest error: " & $GetLastError()
+          )
+      else:
+        raise newException(
+          PuppyError, "WinHttpSendRequest error: " & $GetLastError()
+        )
 
     if WinHttpReceiveResponse(hRequest, nil) == 0:
       raise newException(
@@ -139,7 +174,6 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
       )
 
     result.code = statusCode
-
     var
       responseHeaderBytes: DWORD
       responseHeaderBuf: string
@@ -202,14 +236,12 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
         raise newException(
           PuppyError, "WinHttpReadData error: " & $GetLastError()
         )
-
       i += bytesRead
-
       if bytesRead == 0:
         break
 
       if i == result.body.len:
-        result.body.setLen(min(i * 2, i + 100 * 1024 * 1024))
+        result.body.setLen(i * 2)
 
     result.body.setLen(i)
 
@@ -221,4 +253,4 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
   finally:
     discard WinHttpCloseHandle(hRequest)
     discard WinHttpCloseHandle(hConnect)
-    discard WinHttpCloseHandle(hSession)
+    discard WinHttpCloseHandle(hSession)

src/puppy/platforms/win32/windefs.nim

@@ -4,7 +4,6 @@ when defined(cpu64):
 else:
   type
     ULONG_PTR* = uint32
-
 type
   BOOL* = int32
   LPBOOL* = ptr BOOL
@@ -22,7 +21,6 @@ type
   INTERNET_PORT* = WORD
   DWORD_PTR* = ULONG_PTR
   LPVOID* = pointer
-
 const
   CP_UTF8* = 65001
   WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY* = 4
@@ -33,8 +31,17 @@ const
   WINHTTP_QUERY_FLAG_NUMBER* = 0x20000000
   WINHTTP_QUERY_RAW_HEADERS_CRLF* = 22
   ERROR_INSUFFICIENT_BUFFER* = 122
+  ERROR_WINHTTP_SECURE_FAILURE* = 12175
+  ERROR_INTERNET_INVALID_CA* = 12045
+  WINHTTP_OPTION_SECURITY_FLAGS* = 31
+  SECURITY_FLAG_IGNORE_UNKNOWN_CA* = 0x00000100
+  # SECURITY_FLAG_IGNORE_WRONG_USAGE* = 0x00000200
+  SECURITY_FLAG_IGNORE_CERT_WRONG_USAGE* = 0x00000200
+  SECURITY_FLAG_IGNORE_CERT_CN_INVALID* = 0x00001000
+  SECURITY_FLAG_IGNORE_CERT_DATE_INVALID* = 0x00002000
 
 {.push importc, stdcall.}
+{.push importc, stdcall.}
 
 proc GetLastError*(): DWORD {.dynlib: "kernel32".}
 
@@ -88,6 +95,13 @@ proc WinHttpOpenRequest*(
   dwFlags: DWORD
 ): HINTERNET {.dynlib: "winhttp".}
 
+proc WinHttpSetOption* (
+  hInternet: HINTERNET,
+  dwOption: DWORD,
+  lpBuffer: LPVOID,
+  dwBufferLength: DWORD
+): BOOL {.dynlib: "winhttp".}
+
 proc WinHttpAddRequestHeaders*(
   hRequest: HINTERNET,
   lpszHeaders: LPCWSTR,
@@ -127,5 +141,4 @@ proc WinHttpReadData*(
 ): BOOL {.dynlib: "winhttp".}
 
 proc WinHttpCloseHandle*(hInternet: HINTERNET): BOOL {.dynlib: "winhttp".}
-
-{.pop.}
+{.pop.}

tests/data/ssl.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDazCCAlOgAwIBAgIUL6WPqwJb9MBbrqL3v1gIX42hySMwDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMwNzI1MjFaFw0yMzAy
+MjMwNzI1MjFaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQCqA9pvOjC8Jg7wiX3aiXedg7LKS/V3coINi/Xzh7zY
+1YERUyda/nRpFjmONtJkRipxA59ZZfNfMLoOtl5BhcE4LlLC81m48KQt7/eR3SMx
+tkqFByOA5WltulhXHKghyJhyjy0UDIrBHy5Ic7X8dkR1n9spqpfWyM6VMyDOX9Fl
+8N5H03ioicvjZ4n6Mrc2sCMl39G5GFyUgBX6UAAsPg4xrfPL5Yz+TGSeCV7UtFDc
+nfpR4l5WQERqU4IJCNQj5MxweNZNa00sTOP5s7zuzFiGVQqkCz7FWzpZVUbMggBo
+0UfhCNlnFGPMzddG/IsN0ojCRFzBufFa3jC5AzuhwZk/AgMBAAGjUzBRMB0GA1Ud
+DgQWBBSodOQnZd0tc3IMtnEDIoPzw8tMWzAfBgNVHSMEGDAWgBSodOQnZd0tc3IM
+tnEDIoPzw8tMWzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAm
+9IJBZ0xiQUSsUyBe0bZ+1iBkzX8ci1df1rqGooVIIkF3L5HYq8MME0oi5YdQrpqJ
+I4OArRBwz06SUqMP16KTibOgnc8EYM0ZdhqkdK5dZQnyCBpExtlz73U0ELnMvdGE
+a+1ZsnPKXgYXaEfbWWs5EN6avbAkURuzzt4eg7t4XCxm8T/MJqL7D3xoT7ion1v0
+iXsdbBVoC5A0nvPWdraOvB7PI4Z951bywuj/7VRIPlMYUFmrYbOGo5oJudZHwrl8
+N3vDHkgr5GFeB5cyqlPjUNrfjxTQP6Mim3AtaHC30NhhABmxQVjj53Hn+oDuNeF6
+GSj2EtzNm76AU377GrBi
+-----END CERTIFICATE-----

tests/data/ssl.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQCqA9pvOjC8Jg7w
+iX3aiXedg7LKS/V3coINi/Xzh7zY1YERUyda/nRpFjmONtJkRipxA59ZZfNfMLoO
+tl5BhcE4LlLC81m48KQt7/eR3SMxtkqFByOA5WltulhXHKghyJhyjy0UDIrBHy5I
+c7X8dkR1n9spqpfWyM6VMyDOX9Fl8N5H03ioicvjZ4n6Mrc2sCMl39G5GFyUgBX6
+UAAsPg4xrfPL5Yz+TGSeCV7UtFDcnfpR4l5WQERqU4IJCNQj5MxweNZNa00sTOP5
+s7zuzFiGVQqkCz7FWzpZVUbMggBo0UfhCNlnFGPMzddG/IsN0ojCRFzBufFa3jC5
+AzuhwZk/AgMBAAECggEBAKZMLpUtbg4bi/FsC1Z/sCi6cV++/NNhhiSKCoGy+918
+uUqg85Xl3ygLPTEGHrVGjK0OxgdD2dH6b4OEjp24n068wOc/8Tsc5vqoBpj+nTY7
+AJkuamPiAkX6R/6tYSfqdnNX6Nf7jJ1qSnND+3Z+mGgVfOI8o1jMAoWeBTDYOJJk
+eFc0e2O+vBburjj//daVizq47tEIoVRRd+nmsnaMXFuvZh283W6ds1D63HLLZatS
+Jwbycxj0PHQOSEUmeRbmcO6XJtfPuD6mXumtMUgb0OMQtnIj3tY3sWDEAM4cWrP2
+9HZKNFDkoD+Xb7HXcFY9MfvaMzdB/MP+LKo947fiGAECgYEA1Dy+S0bPSrA9X2hh
+vbXBSZvqQsx2xSti9Ern9J/c9ExDe5jsRLTwONBayQ4ctIeuEKBKtr6dY96cKLEY
+QuB40Qzej3ht0nGhx8S57U6rLugIbwVVrvRfPZ7PDWDTGvK4a3s3hZgvxmzTrYIA
+6ZzsfH01pWSgFJ9/66wci56K8b8CgYEAzRJY/iEvVujwMR7HRF0bDwTYzIMsPJgp
+grKz31fcgUesVttUCMKG0Ge+L515xPFjPG9TgFCFwDaQSomwGm1OwAqYTmA5w+WU
+iFncBgs7KhHx5rfwX9JtiEaouNI3R4po00WGA+zz0UYRjVCJHrf+MyV537VSI4FG
+XrP6uiYkOIECgYEAs31/ref/rXmpHcQITUmmYttCXiXPGGbd9B5ZVu/QDKdmtuOY
+hW7Ebjf/X2PY8PCCTDtTlINWVjzQsjU7gGuYoauRmaJOtpg1Kt58I27RpQTFBSds
+1F6FIXbqQrUtM/Ar+XImfYw8c0JcLrPwk6GL+qhlsy+LloVhyO0w4v89IL8CgYEA
+gkx7IRWix50AKKW+xRBHhhZ1ThS2gdXI4lN7eJiR8c7BkPqQ/XPkRvzz2bs8SMd7
+X0X5D1maclP5AHNV4qS7WcghmAMKEQ+Jfc1iwLBYKlX2lrsezzOcBu+merCPETS/
+gCX3jfz7umfD9T9LsKoFqSfRtTO3efnE5Z2D3M0pTIECgYEAwxe+eDuqKVNEorJn
+/3mN5SzC0kXMOiT3UEgMcPAJaizzNjD7d0oIz7afzgJJggGsnerFRFe3Bmw0GJJ5
+lVyanZW3kumPw1B86kakkiaeEAy3uAD6QPcU6vlSlwREkxAujr/K2sCaNtpYwWly
+SCT+0J/TkI28lbnzl1eSG2aQUbA=
+-----END PRIVATE KEY-----

tests/https_server.py

@@ -0,0 +1,46 @@
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import ssl
+
+host = ("0.0.0.0", 443)
+
+class ExecuteServer(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == "/connect":
+            try:
+                self.send_response(200)
+                self.send_header("Content-type", "text/html")
+                self.end_headers()
+                self.wfile.write(bytes("test", "utf8"))
+            except:
+                pass
+
+    def do_POST(self):
+        if self.path == "/plain":
+            content_len = int(self.headers.get("Content-Length"))
+            post_body = self.rfile.read(content_len)
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+
+            post_body = str(post_body.decode())
+            print(post_body)
+
+    def log_message(self, format, *args):
+        pass
+
+def start_server():
+    server = HTTPServer(host, ExecuteServer)
+    server.socket = ssl.wrap_socket(server.socket,
+                                     server_side=True,
+                                     certfile="tests/data/ssl.crt",
+                                     keyfile="tests/data/ssl.key",
+                                     ssl_version=ssl.PROTOCOL_TLS)
+
+    print(f"Starting server, listening on port {host[1]}")
+    server.serve_forever()
+
+def main():
+    start_server()
+
+if __name__ == "__main__":
+    main()