FIX: [PERF] Handle utf16 strings using simdutf and std::u16string

mssql_python/pybind/CMakeLists.txt

@@ -215,6 +215,25 @@ endif()
 
 message(STATUS "Final Python library directory: ${PYTHON_LIB_DIR}")
 
+find_package(simdutf CONFIG QUIET)
+
+if(NOT simdutf_FOUND)
+    include(FetchContent)
+    message(STATUS "simdutf not found via find_package; downloading v8.2.0 source archive with FetchContent")
+    set(simdutf_fetchcontent_args
+        URL https://github.com/simdutf/simdutf/archive/refs/tags/v8.2.0.tar.gz
+        URL_HASH SHA256=033a91b1d7d1cb818c1eff49e61faaa1b64a3a530d59ef9efef0195e56bda8b1
+    )
+    if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24")
+        list(APPEND simdutf_fetchcontent_args DOWNLOAD_EXTRACT_TIMESTAMP FALSE)
+    endif()
+    FetchContent_Declare(simdutf ${simdutf_fetchcontent_args})
+    set(SIMDUTF_TESTS OFF CACHE BOOL "Disable simdutf tests" FORCE)
+    set(SIMDUTF_TOOLS OFF CACHE BOOL "Disable simdutf tools" FORCE)
+    set(SIMDUTF_BENCHMARKS OFF CACHE BOOL "Disable simdutf benchmarks" FORCE)
+    FetchContent_MakeAvailable(simdutf)
+endif()
+
 set(DDBC_SOURCE "ddbc_bindings.cpp")
 message(STATUS "Using standard source file: ${DDBC_SOURCE}")
 # Include connection module and logger bridge
@@ -293,6 +312,8 @@ else()
     endif()
 endif()
 
+target_link_libraries(ddbc_bindings PRIVATE simdutf::simdutf)
+
 # Compiler definitions
 target_compile_definitions(ddbc_bindings PRIVATE 
     HAVE_SNPRINTF

mssql_python/pybind/connection/connection.cpp

@@ -45,7 +45,7 @@ static SqlHandlePtr getEnvHandle() {
 // This class wraps low-level ODBC operations like connect/disconnect,
 // transaction control, and autocommit configuration.
 //-------------------------------------------------------------------------------------------------
-Connection::Connection(const std::wstring& conn_str, bool use_pool)
+Connection::Connection(const std::u16string& conn_str, bool use_pool)
     : _connStr(conn_str), _autocommit(false), _fromPool(use_pool) {
     allocateDbcHandle();
 }
@@ -74,17 +74,7 @@ void Connection::connect(const py::dict& attrs_before) {
             setAutocommit(_autocommit);
         }
     }
-    SQLWCHAR* connStrPtr;
-#if defined(__APPLE__) || defined(__linux__)  // macOS/Linux handling
-    LOG("Creating connection string buffer for macOS/Linux");
-    std::vector<SQLWCHAR> connStrBuffer = WStringToSQLWCHAR(_connStr);
-    // Ensure the buffer is null-terminated
-    LOG("Connection string buffer size=%zu", connStrBuffer.size());
-    connStrPtr = connStrBuffer.data();
-    LOG("Connection string buffer created");
-#else
-    connStrPtr = const_cast<SQLWCHAR*>(_connStr.c_str());
-#endif
+    SQLWCHAR* connStrPtr = reinterpretU16stringAsSqlWChar(_connStr);
     SQLRETURN ret;
     {
         // Release the GIL during the blocking ODBC connect call.
@@ -180,8 +170,7 @@ void Connection::disconnect() {
 void Connection::checkError(SQLRETURN ret) const {
     if (!SQL_SUCCEEDED(ret)) {
         ErrorInfo err = SQLCheckError_Wrap(SQL_HANDLE_DBC, _dbcHandle, ret);
-        std::string errorMsg = WideToUTF8(err.ddbcErrorMsg);
-        ThrowStdException(errorMsg);
+        ThrowStdException(err.ddbcErrorMsg);
     }
 }
 
@@ -308,39 +297,13 @@ SQLRETURN Connection::setAttribute(SQLINTEGER attribute, py::object value) {
         return ret;
     } else if (py::isinstance<py::str>(value)) {
         try {
-            std::string utf8_str = value.cast<std::string>();
-
-            // Convert to wide string
-            std::wstring wstr = Utf8ToWString(utf8_str);
-            if (wstr.empty() && !utf8_str.empty()) {
-                LOG("Failed to convert string value to wide string for "
-                    "attribute=%d",
-                    attribute);
-                return SQL_ERROR;
-            }
-            this->wstrStringBuffer.clear();
-            this->wstrStringBuffer = std::move(wstr);
+            this->wstrStringBuffer = value.cast<std::u16string>();
 
             SQLPOINTER ptr;
             SQLINTEGER length;
-
-#if defined(__APPLE__) || defined(__linux__)
-            // For macOS/Linux, convert wstring to SQLWCHAR buffer
-            std::vector<SQLWCHAR> sqlwcharBuffer = WStringToSQLWCHAR(this->wstrStringBuffer);
-            if (sqlwcharBuffer.empty() && !this->wstrStringBuffer.empty()) {
-                LOG("Failed to convert wide string to SQLWCHAR buffer for "
-                    "attribute=%d",
-                    attribute);
-                return SQL_ERROR;
-            }
-
-            ptr = sqlwcharBuffer.data();
-            length = static_cast<SQLINTEGER>(sqlwcharBuffer.size() * sizeof(SQLWCHAR));
-#else
-            // On Windows, wchar_t and SQLWCHAR are the same size
-            ptr = const_cast<SQLWCHAR*>(this->wstrStringBuffer.c_str());
+
+            ptr = reinterpretU16stringAsSqlWChar(this->wstrStringBuffer);
             length = static_cast<SQLINTEGER>(this->wstrStringBuffer.length() * sizeof(SQLWCHAR));
-#endif
 
             SQLRETURN ret = SQLSetConnectAttr_ptr(_dbcHandle->get(), attribute, ptr, length);
             if (!SQL_SUCCEEDED(ret)) {
@@ -442,10 +405,9 @@ std::chrono::steady_clock::time_point Connection::lastUsed() const {
     return _lastUsed;
 }
 
-ConnectionHandle::ConnectionHandle(const std::string& connStr, bool usePool,
+ConnectionHandle::ConnectionHandle(const std::u16string& connStr, bool usePool,
                                    const py::dict& attrsBefore)
-    : _usePool(usePool) {
-    _connStr = Utf8ToWString(connStr);
+    : _usePool(usePool), _connStr(connStr) {
     if (_usePool) {
         _conn = ConnectionPoolManager::getInstance().acquireConnection(_connStr, attrsBefore);
     } else {
@@ -586,9 +548,7 @@ void ConnectionHandle::setAttr(int attribute, py::object value) {
             std::string errorMsg =
                 "Failed to set connection attribute " + std::to_string(attribute);
             if (!errorInfo.ddbcErrorMsg.empty()) {
-                // Convert wstring to string for concatenation
-                std::string ddbcErrorStr = WideToUTF8(errorInfo.ddbcErrorMsg);
-                errorMsg += ": " + ddbcErrorStr;
+                errorMsg += ": " + errorInfo.ddbcErrorMsg;
             }
 
             LOG("Connection setAttribute failed: %s", errorMsg.c_str());

mssql_python/pybind/connection/connection.h

@@ -21,7 +21,7 @@
 
 class Connection {
   public:
-    Connection(const std::wstring& connStr, bool fromPool);
+    Connection(const std::u16string& connStr, bool fromPool);
 
     ~Connection();
 
@@ -63,12 +63,12 @@ class Connection {
     void checkError(SQLRETURN ret) const;
     void applyAttrsBefore(const py::dict& attrs_before);
 
-    std::wstring _connStr;
+    std::u16string _connStr;
     bool _fromPool = false;
     bool _autocommit = true;
     SqlHandlePtr _dbcHandle;
     std::chrono::steady_clock::time_point _lastUsed;
-    std::wstring wstrStringBuffer;  // wstr buffer for string attribute setting
+    std::u16string wstrStringBuffer;  // UTF-16 buffer for wide ODBC attributes
     std::string strBytesBuffer;     // string buffer for byte attributes setting
 
     // Track child statement handles to mark them as implicitly freed when connection closes
@@ -90,7 +90,7 @@ class Connection {
 
 class ConnectionHandle {
   public:
-    ConnectionHandle(const std::string& connStr, bool usePool,
+    ConnectionHandle(const std::u16string& connStr, bool usePool,
                      const py::dict& attrsBefore = py::dict());
     ~ConnectionHandle();
 
@@ -108,5 +108,5 @@ class ConnectionHandle {
   private:
     std::shared_ptr<Connection> _conn;
     bool _usePool;
-    std::wstring _connStr;
+    std::u16string _connStr;
 };

mssql_python/pybind/connection/connection_pool.cpp

@@ -12,7 +12,7 @@
 ConnectionPool::ConnectionPool(size_t max_size, int idle_timeout_secs)
     : _max_size(max_size), _idle_timeout_secs(idle_timeout_secs), _current_size(0) {}
 
-std::shared_ptr<Connection> ConnectionPool::acquire(const std::wstring& connStr,
+std::shared_ptr<Connection> ConnectionPool::acquire(const std::u16string& connStr,
                                                     const py::dict& attrs_before) {
     std::vector<std::shared_ptr<Connection>> to_disconnect;
     std::shared_ptr<Connection> valid_conn = nullptr;
@@ -145,7 +145,7 @@ ConnectionPoolManager& ConnectionPoolManager::getInstance() {
     return manager;
 }
 
-std::shared_ptr<Connection> ConnectionPoolManager::acquireConnection(const std::wstring& connStr,
+std::shared_ptr<Connection> ConnectionPoolManager::acquireConnection(const std::u16string& connStr,
                                                                      const py::dict& attrs_before) {
     std::shared_ptr<ConnectionPool> pool;
     {
@@ -163,7 +163,7 @@ std::shared_ptr<Connection> ConnectionPoolManager::acquireConnection(const std::
     return pool->acquire(connStr, attrs_before);
 }
 
-void ConnectionPoolManager::returnConnection(const std::wstring& conn_str,
+void ConnectionPoolManager::returnConnection(const std::u16string& conn_str,
                                              const std::shared_ptr<Connection> conn) {
     std::shared_ptr<ConnectionPool> pool;
     {

mssql_python/pybind/connection/connection_pool.h

@@ -20,7 +20,7 @@ class ConnectionPool {
     ConnectionPool(size_t max_size, int idle_timeout_secs);
 
     // Acquires a connection from the pool or creates a new one if under limit
-    std::shared_ptr<Connection> acquire(const std::wstring& connStr,
+    std::shared_ptr<Connection> acquire(const std::u16string& connStr,
                                         const py::dict& attrs_before = py::dict());
 
     // Returns a connection to the pool for reuse
@@ -46,11 +46,11 @@ class ConnectionPoolManager {
     void configure(int max_size, int idle_timeout);
 
     // Gets a connection from the appropriate pool (creates one if none exists)
-    std::shared_ptr<Connection> acquireConnection(const std::wstring& conn_str,
+    std::shared_ptr<Connection> acquireConnection(const std::u16string& conn_str,
                                                   const py::dict& attrs_before = py::dict());
 
     // Returns a connection to its original pool
-    void returnConnection(const std::wstring& conn_str, std::shared_ptr<Connection> conn);
+    void returnConnection(const std::u16string& conn_str, std::shared_ptr<Connection> conn);
 
     // Closes all pools and their connections
     void closePools();
@@ -60,7 +60,7 @@ class ConnectionPoolManager {
     ~ConnectionPoolManager() = default;
 
     // Map from connection string to connection pool
-    std::unordered_map<std::wstring, std::shared_ptr<ConnectionPool>> _pools;
+    std::unordered_map<std::u16string, std::shared_ptr<ConnectionPool>> _pools;
 
     // Protects access to the _pools map
     std::mutex _manager_mutex;