Add OpenSSL 3 support (#19814)

* Minor refactor

* Add OpenSSL 3 support

Remove symbols noOpenSSLHacksq and openssl10

* Drop loading of older openssl versions

* Add library path

* Use only versioned libssl soname os OSX

* Update .github/workflows/ci_packages.yml

Co-authored-by: Hein Thant <official.heinthanth@gmail.com>

* On Mac OS X CI, link OpenSSL in /usr/local/lib/

* Install OpenSSL on Mac OS X on azure pipeline

* Remove DYLD_LIBRARY_PATH

Co-authored-by: Hein Thant <official.heinthanth@gmail.com>

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>
Co-authored-by: Hein Thant <official.heinthanth@gmail.com>

.github/workflows/ci_packages.yml

@@ -42,7 +42,10 @@ jobs:
               valgrind libc6-dbg libblas-dev xorg-dev
       - name: 'Install dependencies (macOS)'
         if: runner.os == 'macOS'
-        run: brew install boehmgc make sfml gtk+3
+        run: |
+          brew install boehmgc make sfml gtk+3 openssl@1.1
+          ln -s $(brew --prefix)/opt/openssl/lib/libcrypto.1.1.dylib /usr/local/lib
+          ln -s $(brew --prefix)/opt/openssl/lib/libssl.1.1.dylib /usr/local/lib/
       - name: 'Install dependencies (Windows)'
         if: runner.os == 'Windows'
         shell: bash
@@ -66,4 +69,5 @@ jobs:
 
       - name: 'koch, Run CI'
         shell: bash
-        run: . ci/funs.sh && nimInternalBuildKochAndRunCI
+        run: |
+          . ci/funs.sh && nimInternalBuildKochAndRunCI

azure-pipelines.yml

@@ -130,6 +130,13 @@ jobs:
       displayName: 'Install dependencies (OSX)'
       condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Darwin'))
 
+    - bash: |
+        brew install openssl@1.1
+        ln -s $(brew --prefix)/opt/openssl/lib/libcrypto.1.1.dylib /usr/local/lib
+        ln -s $(brew --prefix)/opt/openssl/lib/libssl.1.1.dylib /usr/local/lib/
+      displayName: 'Install OpenSSL (OSX)'
+      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Darwin'))
+
     - bash: |
         set -e
         . ci/funs.sh

lib/pure/net.nim

@@ -543,18 +543,14 @@ proc fromSockAddr*(sa: Sockaddr_storage | SockAddr | Sockaddr_in | Sockaddr_in6,
   fromSockAddrAux(cast[ptr Sockaddr_storage](unsafeAddr sa), sl, address, port)
 
 when defineSsl:
-  CRYPTO_malloc_init()
-  doAssert SslLibraryInit() == 1
-  SSL_load_error_strings()
-  ERR_load_BIO_strings()
-  OpenSSL_add_all_algorithms()
+  # OpenSSL >= 1.1.0 does not need explicit init.
 
   proc sslHandle*(self: Socket): SslPtr =
     ## Retrieve the ssl pointer of `socket`.
     ## Useful for interfacing with `openssl`.
     self.sslHandle
 
-  proc raiseSSLError*(s = "") =
+  proc raiseSSLError*(s = "") {.raises: [SslError].}=
     ## Raises a new SSL error.
     if s != "":
       raise newException(SslError, s)
@@ -619,9 +615,7 @@ when defineSsl:
                    caDir = "", caFile = ""): SslContext =
     ## Creates an SSL context.
     ##
-    ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1
-    ## are available with the addition of `protSSLv23` which allows for
-    ## compatibility with all of them.
+    ## protVersion is currently unsed.
     ##
     ## There are three options for verify mode:
     ## `CVerifyNone`: certificates are not verified;
@@ -648,16 +642,12 @@ when defineSsl:
     ## or using ECDSA:
     ## - `openssl ecparam -out mykey.pem -name secp256k1 -genkey`
     ## - `openssl req -new -key mykey.pem -x509 -nodes -days 365 -out mycert.pem`
-    var newCTX: SslCtx
-    case protVersion
-    of protSSLv23:
-      newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support.
-    of protSSLv2:
-      raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23")
-    of protSSLv3:
-      raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23")
-    of protTLSv1:
-      newCTX = SSL_CTX_new(TLSv1_method())
+    let mtd = TLS_method()
+    if mtd == nil:
+      raiseSSLError("Failed to create TLS context")
+    var newCTX = SSL_CTX_new(mtd)
+    if newCTX == nil:
+      raiseSSLError("Failed to create TLS context")
 
     if newCTX.SSL_CTX_set_cipher_list(cipherList) != 1:
       raiseSSLError()
@@ -812,24 +802,28 @@ when defineSsl:
     if SSL_set_fd(socket.sslHandle, socket.fd) != 1:
       raiseSSLError()
 
-  proc checkCertName(socket: Socket, hostname: string) =
+  proc checkCertName(socket: Socket, hostname: string) {.raises: [SslError], tags:[RootEffect].} =
     ## Check if the certificate Subject Alternative Name (SAN) or Subject CommonName (CN) matches hostname.
     ## Wildcards match only in the left-most label.
     ## When name starts with a dot it will be matched by a certificate valid for any subdomain
     when not defined(nimDisableCertificateValidation) and not defined(windows):
       assert socket.isSsl
-      let certificate = socket.sslHandle.SSL_get_peer_certificate()
-      if certificate.isNil:
-        raiseSSLError("No SSL certificate found.")
-
-      const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint
-      # https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html
-      let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint,
-        X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, nil)
-      # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_certificate.html
-      X509_free(certificate)
-      if match != 1:
-        raiseSSLError("SSL Certificate check failed.")
+      try:
+        let certificate = socket.sslHandle.SSL_get_peer_certificate()
+        if certificate.isNil:
+          raiseSSLError("No SSL certificate found.")
+
+        const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint
+        # https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html
+        let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint,
+          X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, nil)
+        # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_certificate.html
+        X509_free(certificate)
+        if match != 1:
+          raiseSSLError("SSL Certificate check failed.")
+
+      except LibraryError:
+        raiseSSLError("SSL import failed")
 
   proc wrapConnectedSocket*(ctx: SslContext, socket: Socket,
                             handshake: SslHandshakeType,
@@ -856,6 +850,7 @@ when defineSsl:
       let ret = SSL_connect(socket.sslHandle)
       socketError(socket, ret)
       when not defined(nimDisableCertificateValidation) and not defined(windows):
+        # FIXME: this should be skipped on CVerifyNone
         if hostname.len > 0 and not isIpAddress(hostname):
           socket.checkCertName(hostname)
     of handshakeAsServer:
@@ -1311,7 +1306,7 @@ when defined(nimdoc) or (defined(posix) and not useNimNetLite):
           (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32:
         raiseOSError(osLastError())
 
-when defined(ssl):
+when defineSsl:
   proc gotHandshake*(socket: Socket): bool =
     ## Determines whether a handshake has occurred between a client (`socket`)
     ## and the server that `socket` is connected to.
@@ -1998,7 +1993,7 @@ proc dial*(address: string, port: Port,
     raise newException(IOError, "Couldn't resolve address: " & address)
 
 proc connect*(socket: Socket, address: string,
-    port = Port(0)) {.tags: [ReadIOEffect].} =
+    port = Port(0)) {.tags: [ReadIOEffect, RootEffect].} =
   ## Connects socket to `address`:`port`. `Address` can be an IP address or a
   ## host name. If `address` is a host name, this function will try each IP
   ## of that host name. `htons` is already performed on `port` so you must
@@ -2073,7 +2068,7 @@ proc connectAsync(socket: Socket, name: string, port = Port(0),
   if not success: raiseOSError(lastError)
 
 proc connect*(socket: Socket, address: string, port = Port(0),
-    timeout: int) {.tags: [ReadIOEffect, WriteIOEffect].} =
+    timeout: int) {.tags: [ReadIOEffect, WriteIOEffect, RootEffect].} =
   ## Connects to server as specified by `address` on port specified by `port`.
   ##
   ## The `timeout` parameter specifies the time in milliseconds to allow for