HTTPCLIENT-2356 Extend AuthScheme API and Authentication Logic to Enable SPNEGO Mutual Authentication

httpclient5-testing/src/test/java/org/apache/hc/client5/testing/extension/async/StandardTestClientBuilder.java

@@ -33,6 +33,7 @@
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.client5.http.UserTokenHandler;
 import org.apache.hc.client5.http.auth.AuthSchemeFactory;
+import org.apache.hc.client5.http.auth.CredentialsProvider;
 import org.apache.hc.client5.http.config.ConnectionConfig;
 import org.apache.hc.client5.http.config.TlsConfig;
 import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
@@ -159,6 +160,12 @@ public TestAsyncClientBuilder setDefaultAuthSchemeRegistry(final Lookup<AuthSche
         return this;
     }
 
+    @Override
+    public TestAsyncClientBuilder setDefaultCredentialsProvider(final CredentialsProvider credentialsProvider) {
+        this.clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+        return this;
+    }
+
     @Override
     public TestAsyncClient build() throws Exception {
         final PoolingAsyncClientConnectionManager connectionManager = connectionManagerBuilder

httpclient5-testing/src/test/java/org/apache/hc/client5/testing/extension/async/TestAsyncClientBuilder.java

@@ -33,6 +33,7 @@
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.client5.http.UserTokenHandler;
 import org.apache.hc.client5.http.auth.AuthSchemeFactory;
+import org.apache.hc.client5.http.auth.CredentialsProvider;
 import org.apache.hc.client5.http.config.TlsConfig;
 import org.apache.hc.core5.http.Header;
 import org.apache.hc.core5.http.HttpRequestInterceptor;
@@ -105,6 +106,10 @@ default TestAsyncClientBuilder setDefaultAuthSchemeRegistry(Lookup<AuthSchemeFac
         throw new UnsupportedOperationException("Operation not supported by " + getProtocolLevel());
     }
 
+    default TestAsyncClientBuilder setDefaultCredentialsProvider(CredentialsProvider credentialsProvider) {
+        throw new UnsupportedOperationException("Operation not supported by " + getProtocolLevel());
+    }
+
     TestAsyncClient build() throws Exception;
 
 }

httpclient5-testing/src/test/java/org/apache/hc/client5/testing/extension/sync/StandardTestClientBuilder.java

@@ -33,6 +33,7 @@
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.client5.http.UserTokenHandler;
 import org.apache.hc.client5.http.auth.AuthSchemeFactory;
+import org.apache.hc.client5.http.auth.CredentialsProvider;
 import org.apache.hc.client5.http.classic.ExecChainHandler;
 import org.apache.hc.client5.http.config.ConnectionConfig;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
@@ -150,6 +151,12 @@ public TestClientBuilder addExecInterceptorLast(final String name, final ExecCha
         return this;
     }
 
+    @Override
+    public TestClientBuilder setDefaultCredentialsProvider(final CredentialsProvider credentialsProvider) {
+        this.clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+        return this;
+    }
+
     @Override
     public TestClient build() throws Exception {
         final HttpClientConnectionManager connectionManagerCopy = connectionManager != null ? connectionManager :

httpclient5-testing/src/test/java/org/apache/hc/client5/testing/extension/sync/TestClientBuilder.java

@@ -33,6 +33,7 @@
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.client5.http.UserTokenHandler;
 import org.apache.hc.client5.http.auth.AuthSchemeFactory;
+import org.apache.hc.client5.http.auth.CredentialsProvider;
 import org.apache.hc.client5.http.classic.ExecChainHandler;
 import org.apache.hc.client5.http.io.HttpClientConnectionManager;
 import org.apache.hc.core5.http.Header;
@@ -98,6 +99,10 @@ default TestClientBuilder addExecInterceptorLast(String name, ExecChainHandler i
         throw new UnsupportedOperationException("Operation not supported by " + getProtocolLevel());
     }
 
+    default TestClientBuilder setDefaultCredentialsProvider(CredentialsProvider credentialsProvider) {
+        throw new UnsupportedOperationException("Operation not supported by " + getProtocolLevel());
+    }
+
     TestClient build() throws Exception;
 
 }

httpclient5/src/main/java/org/apache/hc/client5/http/auth/AuthExchange.java

@@ -38,6 +38,9 @@
  */
 public class AuthExchange {
 
+    // This only tracks the server state. In particular, even if the state is SUCCESS,
+    // the authentication may still fail if the challenge sent with an authorized response cannot
+    // be validated locally for AuthScheme2 schemes.
     public enum State {
 
         UNCHALLENGED, CHALLENGED, HANDSHAKE, FAILURE, SUCCESS

httpclient5/src/main/java/org/apache/hc/client5/http/auth/AuthScheme.java

@@ -86,6 +86,10 @@
  * containing the terminal authorization response, the scheme is considered unsuccessful
  * and in FAILED state.
  * </p>
+ * <p>
+ * This interface cannot correctly handle some authentication methods, like SPNEGO.
+ * See {@link AuthScheme2} for a more capable interface.
+ * </p>
  *
  * @since 4.0
  */
@@ -128,6 +132,9 @@ void processChallenge(
      * successfully or unsuccessfully), that is, all the required authorization
      * challenges have been processed in their entirety.
      *
+     * Note that due to some assumptions made about the control flow by the authentication code
+     * returning true will immediately cause the authentication process to fail.
+     *
      * @return {@code true} if the authentication process has been completed,
      * {@code false} otherwise.
      *

httpclient5/src/main/java/org/apache/hc/client5/http/auth/AuthScheme2.java

@@ -0,0 +1,102 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+package org.apache.hc.client5.http.auth;
+
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.HttpRequest;
+import org.apache.hc.core5.http.protocol.HttpContext;
+
+/**
+ * This is an improved version of the {@link AuthScheme} interface, amended to be able to handle
+ * a conversation involving multiple challenge-response transactions and adding the ability to check
+ * the results of a final token sent together with the successful HTTP request as required by
+ * RFC 4559 and RFC 7546.
+ *
+ * @since 5.5
+ */
+public interface AuthScheme2 extends AuthScheme {
+
+    /**
+     * Processes the given auth challenge. Some authentication schemes may involve multiple
+     * challenge-response exchanges. Such schemes must be able to maintain internal state
+     * when dealing with sequential challenges.
+     *
+     * The {@link AuthScheme} interface  implicitly assumes that that the token passed here is
+     * simply stored in this method, and the actual authentication takes place in
+     * {@link org.apache.hc.client5.http.auth.AuthScheme#generateAuthResponse(HttpHost, HttpRequest, HttpContext) generateAuthResponse }
+     * and/or {@link org.apache.hc.client5.http.auth.AuthScheme#isResponseReady(HttpHost, HttpRequest, HttpContext) generateAuthResponse },
+     * as only those methods receive the HttpHost, and only those can throw an
+     * AuthenticationException.
+     *
+     * This new methods signature makes it possible to process the token and throw an
+     * AuthenticationException immediately even when no response is sent (i.e. processing an SPNEGO or SCRAM mutual
+     * authentication response)
+     *
+     * When {@link  isChallengeExpected} returns true, but no challenge was sent, then this method must
+     * be called with a null {@link AuthChallenge} so that the Scheme can handle this situation.
+     *
+     * @param host HTTP host
+     * @param authChallenge the auth challenge or null if no challenge was received
+     * @param context HTTP context
+     * @param challenged true if the response was unauthorised (401/407)
+     * @throws AuthenticationException in case the authentication process is unsuccessful.
+     * @since 5.5
+     */
+    void processChallenge(
+            HttpHost host,
+            AuthChallenge authChallenge,
+            HttpContext context,
+            boolean challenged) throws AuthenticationException;
+
+    /**
+     * The old processChallenge signature is unfit for use in AuthScheme2.
+     * If the old signature is sufficient for a scheme, then it should implement {@link AuthScheme}
+     * instead AuthScheme2.
+     */
+    @Override
+    default void processChallenge(
+            AuthChallenge authChallenge,
+            HttpContext context) throws MalformedChallengeException {
+        throw new UnsupportedOperationException("on AuthScheme2 implementations only the four "
+                + "argument processChallenge method can be called");
+    }
+
+    /**
+     * Indicates that the even authorized (i.e. not 401 or 407) responses must be processed
+     * by this Scheme.
+     *
+     * The original AuthScheme interface only processes unauthorised responses.
+     * This method indicates that non unauthorised responses are expected to contain challenges
+     * and must be processed by the Scheme.
+     * This is required to implement the SPENGO RFC and Kerberos mutual authentication, and SCRAM.
+     *
+     * @return true if responses with non 401/407 response codes must be processed by the scheme.
+     * @since 5.5
+     */
+    boolean isChallengeExpected();
+
+}

httpclient5/src/main/java/org/apache/hc/client5/http/impl/DefaultAuthenticationStrategy.java

@@ -72,6 +72,10 @@ public class DefaultAuthenticationStrategy implements AuthenticationStrategy {
                 StandardAuthScheme.DIGEST,
                 StandardAuthScheme.BASIC));
 
+    protected List<String> getSchemePriority() {
+        return DEFAULT_SCHEME_PRIORITY;
+    }
+
     @Override
     public List<AuthScheme> select(
             final ChallengeType challengeType,
@@ -95,7 +99,7 @@ public List<AuthScheme> select(
         Collection<String> authPrefs = challengeType == ChallengeType.TARGET ?
                 config.getTargetPreferredAuthSchemes() : config.getProxyPreferredAuthSchemes();
         if (authPrefs == null) {
-            authPrefs = DEFAULT_SCHEME_PRIORITY;
+            authPrefs = getSchemePriority();
         }
         if (LOG.isDebugEnabled()) {
             LOG.debug("{} Authentication schemes in the order of preference: {}", exchangeId, authPrefs);

httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/AsyncConnectExec.java

@@ -43,7 +43,9 @@
 import org.apache.hc.client5.http.async.AsyncExecChainHandler;
 import org.apache.hc.client5.http.async.AsyncExecRuntime;
 import org.apache.hc.client5.http.auth.AuthExchange;
+import org.apache.hc.client5.http.auth.AuthenticationException;
 import org.apache.hc.client5.http.auth.ChallengeType;
+import org.apache.hc.client5.http.auth.MalformedChallengeException;
 import org.apache.hc.client5.http.config.RequestConfig;
 import org.apache.hc.client5.http.impl.auth.AuthCacheKeeper;
 import org.apache.hc.client5.http.impl.auth.HttpAuthenticator;
@@ -515,10 +517,11 @@ private boolean needAuthentication(
             final AuthExchange proxyAuthExchange,
             final HttpHost proxy,
             final HttpResponse response,
-            final HttpClientContext context) {
+            final HttpClientContext context) throws AuthenticationException, MalformedChallengeException {
         final RequestConfig config = context.getRequestConfigOrDefault();
         if (config.isAuthenticationEnabled()) {
             final boolean proxyAuthRequested = authenticator.isChallenged(proxy, ChallengeType.PROXY, response, proxyAuthExchange, context);
+            final boolean proxyMutualAuthRequired = authenticator.isChallengeExpected(proxyAuthExchange);
 
             if (authCacheKeeper != null) {
                 if (proxyAuthRequested) {
@@ -528,7 +531,7 @@ private boolean needAuthentication(
                 }
             }
 
-            if (proxyAuthRequested) {
+            if (proxyAuthRequested || proxyMutualAuthRequired) {
                 final boolean updated = authenticator.updateAuthState(proxy, ChallengeType.PROXY, response,
                         proxyAuthStrategy, proxyAuthExchange, context);
 

httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/AsyncProtocolExec.java

@@ -38,7 +38,9 @@
 import org.apache.hc.client5.http.async.AsyncExecChainHandler;
 import org.apache.hc.client5.http.async.AsyncExecRuntime;
 import org.apache.hc.client5.http.auth.AuthExchange;
+import org.apache.hc.client5.http.auth.AuthenticationException;
 import org.apache.hc.client5.http.auth.ChallengeType;
+import org.apache.hc.client5.http.auth.MalformedChallengeException;
 import org.apache.hc.client5.http.config.RequestConfig;
 import org.apache.hc.client5.http.impl.DefaultSchemePortResolver;
 import org.apache.hc.client5.http.impl.RequestSupport;
@@ -305,11 +307,12 @@ private boolean needAuthentication(
             final HttpHost target,
             final String pathPrefix,
             final HttpResponse response,
-            final HttpClientContext context) {
+            final HttpClientContext context) throws AuthenticationException, MalformedChallengeException {
         final RequestConfig config = context.getRequestConfigOrDefault();
         if (config.isAuthenticationEnabled()) {
             final boolean targetAuthRequested = authenticator.isChallenged(
                     target, ChallengeType.TARGET, response, targetAuthExchange, context);
+            final boolean targetMutualAuthRequired = authenticator.isChallengeExpected(targetAuthExchange);
 
             if (authCacheKeeper != null) {
                 if (targetAuthRequested) {
@@ -321,6 +324,7 @@ private boolean needAuthentication(
 
             final boolean proxyAuthRequested = authenticator.isChallenged(
                     proxy, ChallengeType.PROXY, response, proxyAuthExchange, context);
+            final boolean proxyMutualAuthRequired = authenticator.isChallengeExpected(proxyAuthExchange);
 
             if (authCacheKeeper != null) {
                 if (proxyAuthRequested) {
@@ -330,7 +334,7 @@ private boolean needAuthentication(
                 }
             }
 
-            if (targetAuthRequested) {
+            if (targetAuthRequested || targetMutualAuthRequired) {
                 final boolean updated = authenticator.updateAuthState(target, ChallengeType.TARGET, response,
                         targetAuthStrategy, targetAuthExchange, context);
 
@@ -340,7 +344,7 @@ private boolean needAuthentication(
 
                 return updated;
             }
-            if (proxyAuthRequested) {
+            if (proxyAuthRequested || proxyMutualAuthRequired) {
                 final boolean updated = authenticator.updateAuthState(proxy, ChallengeType.PROXY, response,
                         proxyAuthStrategy, proxyAuthExchange, context);