TLS 1.2 handshake fails

[rychale]

Describe the bug
Connecting to SSL socket using TLS 1.2 returns an error: {:error, {:tls_alert, {:handshake_failure, ~c"TLS server: In state hello at tls_handshake.erl:269 generated SERVER ALERT: Fatal - Handshake Failure\n malformed_handshake_data"}}}.

Doing ssl.versions reports both TLS 1.2 and 1.3 are supported:

SSL version [ssl_app: ~c"11.0.2", supported: [:"tlsv1.3", :"tlsv1.2"], supported_dtls: [:"dtlsv1.2"], available: [:"tlsv1.3", :"tlsv1.2", :"tlsv1.1", :tlsv1], available_dtls: [:"dtlsv1.2", :dtlsv1], implemented: [: "tlsv1.3", :"tlsv1.2", :"tlsv1.1", :tlsv1], implemented_dtls: [:"dtlsv1.2", :dtlsv1]]

To Reproduce

defmodule TlsHandshakeTest do
  use ExUnit.Case, async: false

  setup do
    {:module, :ssl} = Code.ensure_loaded(:ssl)

    IO.puts("SSL version #{inspect(:ssl.versions())}")

    :ok
  end

  test "SSL client" do
    {:ok, sock} =
      :ssl.listen(0,
        active: false,
        reuseaddr: true,
        mode: :binary,
        packet: :raw,                                                                                                                                                                                                         verify: :verify_none,
        versions: [:"tlsv1.2", :"tlsv1.3"]
      )

    {:ok, {_address, port}} = :ssl.sockname(sock)

    task =
      Task.async(fn ->
        {:ok, sock} = :ssl.transport_accept(sock)
        {:ok, sock} = :ssl.handshake(sock, 1_000)
                                                                                                                                                                                                                              {:ok, sock}
      end)

    assert {:ok, conn} =
             :ssl.connect(
               to_charlist("localhost"),
               port,
               [active: false, verify: :verify_none, versions: [:"tlsv1.2"], log_level: :debug],
               1_000
             )

    assert {:ok, _sock} = Task.await(task)                                                                                                                                                                              end
end

Expected behavior
A normal SSL handshake would occur, without crashing the Erlang process and terminating the connection.

Affected versions

Erlang/OTP 26 [erts-14.0.2] [source] [64-bit] [smp:4:4] [ds:4:4:10] [async-threads:1] [jit:ns]

[nix-shell@jane:~/git/tls-handshake]$ erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell
26.0.2

Additional context

[nix-shell@jane:~/git/tls-handshake]$ mix test --no-start test/integration_test.exs
SSL version [ssl_app: ~c"11.0.2", supported: [:"tlsv1.3", :"tlsv1.2"], supported_dtls: [:"dtlsv1.2"], available: [:"tlsv1.3", :"tlsv1.2", :"tlsv1.1", :tlsv1], available_dtls: [:"dtlsv1.2", :dtlsv1], implemented: [:"tlsv1.3", :"tlsv1.2", :"tlsv1.1", :tlsv1], implemented_dtls: [:"dtlsv1.2", :dtlsv1]]
>>> TLS 1.2 Handshake, ClientHello
[{client_version,{3,3}},
 {random,<<100,174,65,159,192,117,62,96,112,144,198,1,226,178,45,154,73,131,
           182,38,28,248,173,218,225,6,110,24,227,136,213,78>>},
 {session_id,<<>>},
 {cookie,undefined},
 {cipher_suites,["TLS_EMPTY_RENEGOTIATION_INFO_SCSV",
                 "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
                 "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
                 "TLS_ECDHE_ECDSA_WITH_AES_256_CCM",
                 "TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8",
                 "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384",
                 "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
                 "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
                 "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
                 "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
                 "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
                 "TLS_ECDHE_ECDSA_WITH_AES_128_CCM",
                 "TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8",
                 "TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384",
                 "TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384",
                 "TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384",
                 "TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384",
                 "TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256",
                 "TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256",
                 "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256",
                 "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
                 "TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256",
                 "TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256",
                 "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384",
                 "TLS_DHE_DSS_WITH_AES_256_GCM_SHA384",
                 "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256",
                 "TLS_DHE_DSS_WITH_AES_256_CBC_SHA256",
                 "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256",
                 "TLS_DHE_DSS_WITH_AES_128_GCM_SHA256",
                 "TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
                 "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256",
                 "TLS_DHE_DSS_WITH_AES_128_CBC_SHA256",
                 "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
                 "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
                 "TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA",
                 "TLS_ECDH_RSA_WITH_AES_256_CBC_SHA",
                 "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA",
                 "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
                 "TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA",
                 "TLS_ECDH_RSA_WITH_AES_128_CBC_SHA",
                 "TLS_DHE_RSA_WITH_AES_256_CBC_SHA",
                 "TLS_DHE_DSS_WITH_AES_256_CBC_SHA",
                 "TLS_DHE_RSA_WITH_AES_128_CBC_SHA",
                 "TLS_DHE_DSS_WITH_AES_128_CBC_SHA"]},
 {compression_methods,[0]},
 {extensions,#{srp => undefined,
               signature_algs =>
                   {hash_sign_algos,[{sha512,ecdsa},
                                     {sha512,rsa_pss_pss},
                                     {sha512,rsa_pss_rsae},
                                     {sha512,rsa},
                                     {sha384,ecdsa},
                                     {sha384,rsa_pss_pss},
                                     {sha384,rsa_pss_rsae},
                                     {sha384,rsa},
                                     {sha256,ecdsa},
                                     {sha256,rsa_pss_pss},
                                     {sha256,rsa_pss_rsae},
                                     {sha256,rsa}]},
               signature_algs_cert => undefined,use_srtp => undefined,
               elliptic_curves =>
                   {elliptic_curves,[{1,3,132,0,39},
                                     {1,3,132,0,38},
                                     {1,3,132,0,35},
                                     {1,3,36,3,3,2,8,1,1,13},
                                     {1,3,132,0,36},
                                     {1,3,132,0,37},
                                     {1,3,36,3,3,2,8,1,1,11},
                                     {1,3,132,0,34},
                                     {1,3,132,0,16},
                                     {1,3,132,0,17},
                                     {1,3,36,3,3,2,8,1,1,7},
                                     {1,3,132,0,10},
                                     {1,2,840,10045,3,1,7}]},
               sni => {sni,"localhost"},
               max_frag_enum => undefined,alpn => undefined,
               ec_point_formats => {ec_point_formats,[0]},
               next_protocol_negotiation => undefined,
               renegotiation_info => {renegotiation_info,undefined}}}]

14:01:04.005 [debug] [message: {:client_hello, {3, 3}, <<100, 174, 65, 159, 192, 117, 62, 96, 112, 144, 198, 1, 226, 178, 45, 154, 73, 131, 182, 38, 28, 248, 173, 218, 225, 6, 110, 24, 227, 136, 213, 78>>, "", :undefined, [<<0, 255>>, <<192, 44>>, <<192, 48>>, <<192, 173>>, <<192, 175>>, <<192, 36>>, <<192, 40>>, "̩", "̨", <<192, 43>>, <<192, 47>>, <<192, 172>>, <<192, 174>>, <<192, 46>>, <<192, 50>>, <<192, 38>>, <<192, 42>>, <<192, 45>>, <<192, 49>>, <<192, 35>>, <<192, 39>>, <<192, 37>>, <<192, 41>>, <<0, 159>>, <<0, 163>>, <<0, 107>>, <<0, 106>>, <<0, 158>>, <<0, 162>>, "̪", <<0, 103>>, <<0, 64>>, <<192, 10>>, <<192, 20>>, <<192, 5>>, <<192, 15>>, <<192, 9>>, <<192, 19>>, <<192, 4>>, <<192, 14>>, <<0, 57>>, <<0, ...>>, <<...>>, ...], [0], %{srp: :undefined, signature_algs: {:hash_sign_algos, [sha512: :ecdsa, sha512: :rsa_pss_pss, sha512: :rsa_pss_rsae, sha512: :rsa, sha384: :ecdsa, sha384: :rsa_pss_pss, sha384: :rsa_pss_rsae, sha384: :rsa, sha256: :ecdsa, sha256: :rsa_pss_pss, sha256: :rsa_pss_rsae, sha256: :rsa]}, signature_algs_cert: :undefined, use_srtp: :undefined, elliptic_curves: {:elliptic_curves, [{1, 3, 132, 0, 39}, {1, 3, 132, 0, 38}, {1, 3, 132, 0, 35}, {1, 3, 36, 3, 3, 2, 8, 1, 1, 13}, {1, 3, 132, 0, 36}, {1, 3, 132, 0, 37}, {1, 3, 36, 3, 3, 2, 8, 1, 1, 11}, {1, 3, 132, 0, 34}, {1, 3, 132, 0, 16}, {1, 3, 132, 0, 17}, {1, 3, 36, 3, 3, 2, 8, 1, 1, 7}, {1, 3, 132, 0, 10}, {1, 2, 840, 10045, 3, 1, 7}]}, sni: {:sni, ~c"localhost"}, max_frag_enum: :undefined, alpn: :undefined, ec_point_formats: {:ec_point_formats, [0]}, next_protocol_negotiation: :undefined, renegotiation_info: {:renegotiation_info, :undefined}}}, protocol: :handshake, direction: :outbound]
writing (224 bytes) TLS 1.2 Record Protocol, handshake
0000 - 16 03 03 00 db 01 00 00  d7 03 03 64 ae 41 9f c0    ...........d.A..
0010 - 75 3e 60 70 90 c6 01 e2  b2 2d 9a 49 83 b6 26 1c    u>`p.....-.I..&.
0020 - f8 ad da e1 06 6e 18 e3  88 d5 4e 00 00 58 00 ff    .....n....N..X..
0030 - c0 2c c0 30 c0 ad c0 af  c0 24 c0 28 cc a9 cc a8    .,.0.....$.(....
0040 - c0 2b c0 2f c0 ac c0 ae  c0 2e c0 32 c0 26 c0 2a    .+./.......2.&.*
0050 - c0 2d c0 31 c0 23 c0 27  c0 25 c0 29 00 9f 00 a3    .-.1.#.'.%.)....
0060 - 00 6b 00 6a 00 9e 00 a2  cc aa 00 67 00 40 c0 0a    .k.j.......g.@..
0070 - c0 14 c0 05 c0 0f c0 09  c0 13 c0 04 c0 0e 00 39    ...............9
0080 - 00 38 00 33 00 32 01 00  00 56 00 0b 00 02 01 00    .8.3.2...V......
0090 - 00 00 00 0e 00 0c 00 00  09 6c 6f 63 61 6c 68 6f    .........localho
00a0 - 73 74 00 0a 00 1c 00 1a  00 0e 00 0d 00 19 00 1c    st..............
00b0 - 00 0b 00 0c 00 1b 00 18  00 09 00 0a 00 1a 00 16    ................
00c0 - 00 17 00 0d 00 1a 00 18  06 03 08 0b 08 06 06 01    ................
00d0 - 05 03 08 0a 08 05 05 01  04 03 08 09 08 04 04 01    ................

14:01:04.016 [debug] [message: [<<22, 3, 3, 0, 219>>, <<1, 0, 0, 215, 3, 3, 100, 174, 65, 159, 192, 117, 62, 96, 112, 144, 198, 1, 226, 178, 45, 154, 73, 131, 182, 38, 28, 248, 173, 218, 225, 6, 110, 24, 227, 136, 213, 78, 0, 0, 88, 0, 255, 192, 44, 192, 48, ...>>], protocol: :record, direction: :outbound]

14:01:04.024 [debug] [message: {:ssl_tls, 21, {3, 3}, <<2, 40>>, false}, protocol: :record, direction: :inbound]
reading (7 bytes) TLS 1.2 Record Protocol, alert
0000 - 15 03 03 00 02 02 28                                ......(

14:01:04.024 [notice] TLS :server: In state :hello at tls_handshake.erl:269 generated SERVER ALERT: Fatal - Handshake Failure
 - :malformed_handshake_data

14:01:04.033 [notice] TLS :client: In state :hello received SERVER ALERT: Fatal - Handshake Failure


14:01:04.026 [error] Task #PID<0.234.0> started from #PID<0.230.0> terminating
** (MatchError) no match of right hand side value: {:error, {:tls_alert, {:handshake_failure, ~c"TLS server: In state hello at tls_handshake.erl:269 generated SERVER ALERT: Fatal - Handshake Failure\n malformed_handshake_data"}}}
    test/integration_test.exs:28: anonymous fn/1 in TlsHandshakeTest."test SSL client"/1
    (elixir 1.15.2) lib/task/supervised.ex:101: Task.Supervised.invoke_mfa/2
    (elixir 1.15.2) lib/task/supervised.ex:36: Task.Supervised.reply/4
Function: #Function<0.61065134/0 in TlsHandshakeTest."test SSL client"/1>
    Args: []


  1) test SSL client (TlsHandshakeTest)
     test/integration_test.exs:12
     ** (EXIT from #PID<0.230.0>) an exception was raised:
         ** (MatchError) no match of right hand side value: {:error, {:tls_alert, {:handshake_failure, ~c"TLS server: In state hello at tls_handshake.erl:269 generated SERVER ALERT: Fatal - Handshake Failure\n malformed_handshake_data"}}}
             test/integration_test.exs:28: anonymous fn/1 in TlsHandshakeTest."test SSL client"/1
             (elixir 1.15.2) lib/task/supervised.ex:101: Task.Supervised.invoke_mfa/2
             (elixir 1.15.2) lib/task/supervised.ex:36: Task.Supervised.reply/4

[IngelaAndin]

Well, this a somewhat intricate error case that I think we could handle better. In TLS-1.2 it is possible to configure the server to use so called anonymous cipher suites (which is not supported by default and considered a testing/debug feature) and in this case it is possible to have a server that is not configured with a certificate and key. In any normal usage of TLS it is not a valid configuration to run a server without a certificate and key. We will look into improving the error handling. I guess what you want to do is to provide your sever with a certificate and key, they are needed even if the client ignores to verify the servers certificate chain!

[rychale]

Thank you for the quick reply. Still, I can't make it work. Am I doing something wrong?

suite = X509.Test.Suite.new()

{:ok, sock} =
   [
     active: false,
     reuseaddr: true,
     mode: :binary,
     packet: :raw,
     verify: :verify_peer,
     versions: [:"tlsv1.2", :"tlsv1.3"],
     cacerts: suite.cacerts,
     key: {:RSAPrivateKey, X509.PrivateKey.to_der(suite.server_key)}
   ]
   |> then(fn options ->
     IO.inspect(options, label: 'server options')
     :ssl.listen(0, options)
end)

The output is:

server options: [
  active: false,
  reuseaddr: true,
  mode: :binary,
  packet: :raw,
  verify: :verify_peer,
  versions: [:"tlsv1.2", :"tlsv1.3"],
  cacerts: [
    <<48, 130, 2, 74, 48, 130, 1, 179, 160, 3, 2, 1, 2, 2, 9, 0, 239, 139, 31,
      127, 221, 159, 111, 211, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1,                                                                                                                                               11, 5, 0, 48, 51, 49, ...>>
  ],
  key: {:RSAPrivateKey,
   <<48, 130, 2, 92, 2, 1, 0, 2, 129, 129, 0, 247, 148, 77, 60, 217, 240, 42,
     118, 225, 201, 71, 140, 92, 122, 130, 247, 6, 138, 13, 210, 201, 206, 20,
     46, 203, 115, 142, 255, 0, ...>>}
]

[IngelaAndin]

You are still missing the cert option the specifying the servers own cert.

[rychale]

I did something like:

test "SSL client" do
    {:ok, sock} =
      [
        active: false,
        reuseaddr: true,
        mode: :binary,
        packet: :raw,
        verify: :verify_none,
        versions: [:"tlsv1.2", :"tlsv1.3"],
        cacertfile: "/tmp/ca-cert.pem",
        certfile: "/tmp/server-cert.pem",
        keyfile: "/tmp/server-key.pem"
      ]
      |> then(fn options ->
        IO.inspect(options, label: 'server options')
        :ssl.listen(0, options)
      end)
    ...

and now it works.

Thank you so much! It wasn't obvious)

Shall I close the issue?

[rychale]

BTW, what is the correct behavior? Just validate that all the options are passed to listen?

[IngelaAndin]

The options handling has been reworked quite a lot. There are quite many options to SSL/TLS and there is quite a lot of legacy and unfortunate option dependency's to handle, so although this is not a bug, I will mark it enhancement and we will improve the error handling for a future release.

[rychale]

Awesome!