Fix for Bug#25554464, CONNECT FAILS WITH NPE WHEN THE SERVER STARTED

WITH CUSTOM COLLATION.
Fix for Bug#100606 (31818423), UNECESARY CALL TO "SET NAMES 'UTF8'
COLLATE 'UTF8_GENERAL_CI'".

Author: soklakov

CHANGES

@@ -3,6 +3,11 @@
 
 Version 8.0.26
 
+  - Fix for Bug#25554464, CONNECT FAILS WITH NPE WHEN THE SERVER STARTED WITH CUSTOM COLLATION.
+
+  - Fix for Bug#100606 (31818423), UNECESARY CALL TO "SET NAMES 'UTF8' COLLATE 'UTF8_GENERAL_CI'".
+    Thanks to Marc Fletcher for his contribution.
+
   - Fix for Bug#102404 (32435618), CONTRIBUTION: ADD TRACK SESSION STATE CHANGE.
     Thanks to William Lee for his contribution.
 

src/main/core-api/java/com/mysql/cj/CharsetMapping.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, version 2.0, as published by the
+ * Free Software Foundation.
+ *
+ * This program is also distributed with certain software (including but not
+ * limited to OpenSSL) that is licensed under separate terms, as designated in a
+ * particular file or component or in included license documentation. The
+ * authors of MySQL hereby grant you an additional permission to link the
+ * program and your derivative works with the separately licensed software that
+ * they have included with MySQL.
+ *
+ * Without limiting anything contained in the foregoing, this file, which is
+ * part of MySQL Connector/J, is also subject to the Universal FOSS Exception,
+ * version 1.0, a copy of which can be found at
+ * http://oss.oracle.com/licenses/universal-foss-exception.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+ */
+
+package com.mysql.cj;
+
+public interface CharsetSettings {
+
+    public static final String CHARACTER_SET_CLIENT = "character_set_client";
+    public static final String CHARACTER_SET_CONNECTION = "character_set_connection";
+    public static final String CHARACTER_SET_RESULTS = "character_set_results";
+    public static final String COLLATION_CONNECTION = "collation_connection";
+
+    /**
+     * <p>
+     * Choose the MySQL collation index for the handshake packet and the corresponding Java encodings for the password and error messages.
+     * </p>
+     * <p>
+     * This index will be sent with HandshakeResponse setting server variables 'character_set_connection', 'collation_connection', 'character_set_client'
+     * and 'character_set_results' which will be used by the server for decoding passwords during the authentication phase and later on, if
+     * no SET NAMES are issued by {@link #configurePostHandshake(boolean)}.
+     * </p>
+     * <p>
+     * It also means that collation index should be set according to:
+     * <ol>
+     * <li>'passwordCharacterEncoding' if it's present, or
+     * <li>'connectionCollation' if it's present, or
+     * <li>'characterEncoding' if it's present
+     * </ol>
+     * otherwise it will be set to utf8mb4_general_ci or utf8mb4_0900_ai_ci depending on server version.
+     * <p>
+     * Since Protocol::HandshakeV10 and Protocol::HandshakeResponse41 has only one byte for the collation it's not possible to use indexes &gt; 255 during the
+     * handshake.
+     * Also, ucs2, utf16, utf16le and utf32 character sets are impermissible here. Connector/J will try to use utf8mb4 instead.
+     * </p>
+     * 
+     * @param reset
+     *            reset the charsets configuration; needed for changeUser call.
+     * 
+     * @return MySQL collation index to be used during the handshake.
+     */
+    int configurePreHandshake(boolean reset);
+
+    /**
+     * Sets up client character set. This must be done before any further communication with the server!
+     * 
+     * The 'collation_connection', 'character_set_client', 'character_set_connection' and 'character_set_results' server variables are set
+     * according to the collation index selected by {@link #configurePreHandshake(boolean)} and sent in the Protocol::HandshakeV10 packet.
+     * Here Connector/J alters these server variables if needed.
+     * 
+     * @param dontCheckServerMatch
+     *            if true then send the SET NAMES query even if server charset already matches the new value; needed for changeUser call.
+     */
+    void configurePostHandshake(boolean dontCheckServerMatch);
+
+    public boolean doesPlatformDbCharsetMatches();
+
+    String getPasswordCharacterEncoding();
+
+    String getErrorMessageEncoding();
+
+    String getMetadataEncoding();
+
+    int getMetadataCollationIndex();
+
+    boolean getRequiresEscapingEncoder();
+
+    String getJavaEncodingForCollationIndex(int collationIndex);
+
+    int getMaxBytesPerChar(String javaCharsetName);
+
+    int getMaxBytesPerChar(Integer charsetIndex, String javaCharsetName);
+
+    Integer getCollationIndexForCollationName(String collationName);
+
+    String getCollationNameForCollationIndex(Integer collationIndex);
+
+    String getMysqlCharsetNameForCollationIndex(Integer collationIndex);
+
+    int getCollationIndexForJavaEncoding(String javaEncoding, ServerVersion version);
+
+    int getCollationIndexForMysqlCharsetName(String charsetName);
+
+    String getJavaEncodingForMysqlCharset(String mysqlCharsetName);
+
+    String getMysqlCharsetForJavaEncoding(String javaEncoding, ServerVersion version);
+
+    boolean isMultibyteCharset(String javaEncodingName);
+}

src/main/core-api/java/com/mysql/cj/CharsetSettings.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2020, Oracle and/or its affiliates.
+ * Copyright (c) 2002, 2021, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms of the GNU General Public License, version 2.0, as published by the
@@ -96,6 +96,8 @@ public class Constants {
     public static final BigDecimal BIG_DECIMAL_MAX_FLOAT_VALUE = BigDecimal.valueOf(Float.MAX_VALUE);
     public static final BigDecimal BIG_DECIMAL_MAX_NEGATIVE_FLOAT_VALUE = BigDecimal.valueOf(-Float.MAX_VALUE);
 
+    public static final int UNSIGNED_BYTE_MAX_VALUE = 255;
+
     /**
      * Prevents instantiation
      */

src/main/core-api/java/com/mysql/cj/Constants.java

@@ -259,6 +259,9 @@ public enum DatabaseTerm {
                 new StringPropertyDefinition(PropertyKey.characterSetResults, DEFAULT_VALUE_NULL_STRING, RUNTIME_MODIFIABLE,
                         Messages.getString("ConnectionProperties.characterSetResults"), "3.0.13", CATEGORY_SESSION, Integer.MIN_VALUE),
 
+                new StringPropertyDefinition(PropertyKey.customCharsetMapping, DEFAULT_VALUE_NULL_STRING, RUNTIME_MODIFIABLE,
+                        Messages.getString("ConnectionProperties.customCharsetMapping"), "8.0.26", CATEGORY_SESSION, Integer.MIN_VALUE),
+
                 new StringPropertyDefinition(PropertyKey.connectionCollation, DEFAULT_VALUE_NULL_STRING, RUNTIME_MODIFIABLE,
                         Messages.getString("ConnectionProperties.connectionCollation"), "3.0.13", CATEGORY_SESSION, Integer.MIN_VALUE),
 

src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java

@@ -100,6 +100,7 @@ public enum PropertyKey {
     connectTimeout("connectTimeout", true), //
     continueBatchOnError("continueBatchOnError", true), //
     createDatabaseIfNotExist("createDatabaseIfNotExist", true), //
+    customCharsetMapping("customCharsetMapping", true), //
     databaseTerm("databaseTerm", true), //
     defaultAuthenticationPlugin("defaultAuthenticationPlugin", true), //
     defaultFetchSize("defaultFetchSize", true), //

src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms of the GNU General Public License, version 2.0, as published by the
@@ -29,60 +29,24 @@
 
 package com.mysql.cj.protocol;
 
-import com.mysql.cj.CharsetMapping;
-import com.mysql.cj.Messages;
-import com.mysql.cj.ServerVersion;
 import com.mysql.cj.conf.PropertySet;
-import com.mysql.cj.exceptions.ExceptionFactory;
 import com.mysql.cj.exceptions.ExceptionInterceptor;
 
 public interface AuthenticationProvider<M extends Message> {
 
     void init(Protocol<M> prot, PropertySet propertySet, ExceptionInterceptor exceptionInterceptor);
 
-    void connect(ServerSession serverSession, String username, String password, String database);
+    void connect(String username, String password, String database);
 
     /**
      * Re-authenticates as the given user and password
      * 
-     * @param serverSession
-     *            {@link ServerSession} object
      * @param username
      *            user name
      * @param password
      *            password
      * @param database
      *            db name
      */
-    void changeUser(ServerSession serverSession, String username, String password, String database);
-
-    String getEncodingForHandshake();
-
-    /**
-     * Get the MySQL collation index for the handshake packet. A single byte will be added to the packet corresponding to the collation index
-     * found for the requested Java encoding name.
-     * 
-     * If the index is &gt; 255 which may be valid at some point in the future, an exception will be thrown. At the time of this implementation
-     * the index cannot be &gt; 255 and only the COM_CHANGE_USER rpc, not the handshake response, can handle a value &gt; 255.
-     * 
-     * @param enc
-     *            The Java encoding name used to lookup the collation index
-     * @param sv
-     *            server version
-     * @return collation index
-     */
-    static byte getCharsetForHandshake(String enc, ServerVersion sv) {
-        int charsetIndex = 0;
-        if (enc != null) {
-            charsetIndex = CharsetMapping.getCollationIndexForJavaEncoding(enc, sv);
-        }
-        if (charsetIndex == 0) {
-            charsetIndex = CharsetMapping.MYSQL_COLLATION_INDEX_utf8;
-        }
-        if (charsetIndex > 255) {
-            throw ExceptionFactory.createException(Messages.getString("MysqlIO.113", new Object[] { enc }));
-        }
-        return (byte) charsetIndex;
-    }
-
+    void changeUser(String username, String password, String database);
 }

src/main/core-api/java/com/mysql/cj/protocol/AuthenticationProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms of the GNU General Public License, version 2.0, as published by the
@@ -127,8 +127,6 @@ public interface Protocol<M extends Message> {
      */
     void changeUser(String user, String password, String database);
 
-    String getPasswordCharacterEncoding();
-
     boolean versionMeetsMinimum(int major, int minor, int subminor);
 
     /**

src/main/core-api/java/com/mysql/cj/protocol/Protocol.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms of the GNU General Public License, version 2.0, as published by the
@@ -42,8 +42,11 @@ public interface ServerCapabilities {
 
     ServerVersion getServerVersion();
 
-    void setServerVersion(ServerVersion serverVersion);
+    long getThreadId();
+
+    void setThreadId(long threadId);
 
     boolean serverSupportsFracSecs();
 
+    int getServerDefaultCollationIndex();
 }

src/main/core-api/java/com/mysql/cj/protocol/ServerCapabilities.java

@@ -32,6 +32,7 @@
 import java.util.Map;
 import java.util.TimeZone;
 
+import com.mysql.cj.CharsetSettings;
 import com.mysql.cj.ServerVersion;
 import com.mysql.cj.exceptions.CJOperationNotSupportedException;
 import com.mysql.cj.exceptions.ExceptionFactory;
@@ -62,8 +63,6 @@ public interface ServerSession {
      */
     public static int TRANSACTION_COMPLETED = 3;
 
-    public static final String LOCAL_CHARACTER_SET_RESULTS = "local.character_set_results";
-
     ServerCapabilities getCapabilities();
 
     void setCapabilities(ServerCapabilities capabilities);
@@ -92,20 +91,6 @@ public interface ServerSession {
 
     void setOldStatusFlags(int statusFlags);
 
-    /**
-     * 
-     * @return Collation index which server provided in handshake greeting packet
-     */
-    int getServerDefaultCollationIndex();
-
-    /**
-     * Stores collation index which server provided in handshake greeting packet.
-     * 
-     * @param serverDefaultCollationIndex
-     *            collation index
-     */
-    void setServerDefaultCollationIndex(int serverDefaultCollationIndex);
-
     /**
      * 
      * @return TRANSACTION_NOT_STARTED, TRANSACTION_IN_PROGRESS, TRANSACTION_STARTED or TRANSACTION_COMPLETED
@@ -153,8 +138,6 @@ public interface ServerSession {
 
     void setServerVariables(Map<String, String> serverVariables);
 
-    boolean characterSetNamesMatches(String mysqlEncodingName);
-
     /**
      * Get the version of the MySQL server we are talking to.
      * 
@@ -174,46 +157,6 @@ public interface ServerSession {
      */
     boolean isVersion(ServerVersion version);
 
-    /**
-     * 
-     * @return the server's default character set name according to collation index from server greeting,
-     *         or value of 'character_set_server' variable if there is no mapping for that index
-     */
-    String getServerDefaultCharset();
-
-    String getErrorMessageEncoding();
-
-    void setErrorMessageEncoding(String errorMessageEncoding);
-
-    int getMaxBytesPerChar(String javaCharsetName);
-
-    int getMaxBytesPerChar(Integer charsetIndex, String javaCharsetName);
-
-    /**
-     * Returns the Java character encoding name for the given MySQL server
-     * collation index
-     * 
-     * @param collationIndex
-     *            collation index
-     * @return the Java character encoding name for the given MySQL server
-     *         collation index
-     */
-    String getEncodingForIndex(int collationIndex);
-
-    void configureCharacterSets();
-
-    String getCharacterSetMetadata();
-
-    void setCharacterSetMetadata(String characterSetMetadata);
-
-    int getMetadataCollationIndex();
-
-    void setMetadataCollationIndex(int metadataCollationIndex);
-
-    String getCharacterSetResultsOnServer();
-
-    void setCharacterSetResultsOnServer(String characterSetResultsOnServer);
-
     /**
      * Is the server configured to use lower-case table names only?
      * 
@@ -231,10 +174,6 @@ public interface ServerSession {
 
     public boolean isServerTruncatesFracSecs();
 
-    long getThreadId();
-
-    public void setThreadId(long threadId);
-
     boolean isAutoCommit();
 
     void setAutoCommit(boolean autoCommit);
@@ -255,4 +194,7 @@ default ServerSessionStateController getServerSessionStateController() {
         throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported");
     }
 
+    CharsetSettings getCharsetSettings();
+
+    void setCharsetSettings(CharsetSettings charsetSettings);
 }