Update to version v4.19.0

.github/workflows/test.yml

@@ -47,8 +47,11 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04, macos-13]
         include:
+          - os: ubuntu-24.04
+            cc: gcc-12
+            cxx: g++-12
           - os: ubuntu-latest
             sanitizer: ASAN
             cc: gcc-10
@@ -111,7 +114,7 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-13]
         test: ['C++', 'GO']
         include:
           - os: ubuntu-latest
@@ -192,7 +195,7 @@ jobs:
   test-pyreindexer:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: [ubuntu-22.04, ubuntu-24.04]
       fail-fast: false
     runs-on: ${{matrix.os}}
     needs: build
@@ -210,7 +213,7 @@ jobs:
         run: |
           if [[ $OS == ubuntu* ]]; then
             sudo ./dependencies.sh
-            python3 -m pip install setuptools
+            python3 -m pip install setuptools build
           else
             ./dependencies.sh
           fi
@@ -239,7 +242,9 @@ jobs:
         with:
           repository: restream/reindexer-py
       - name: Install PyReindexer
-        run: sudo python3 setup.py install
+        run: |
+          python -m build
+          python -m pip install .
       - name: Test PyReindexer
         run: |
           cd pyreindexer

appveyor.yml

@@ -12,11 +12,11 @@ environment:
       TOOLCHAIN: x86_64-8.1.0-posix-seh-rt_v6-rev0
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
 
-    - BUILD_TYPE: Release
-      COMPILER: MinGW
-      PLATFORM: Win32
-      TOOLCHAIN: i686-8.1.0-posix-dwarf-rt_v6-rev0
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+#    - BUILD_TYPE: Release
+#      COMPILER: MinGW
+#      PLATFORM: Win32
+#      TOOLCHAIN: i686-8.1.0-posix-dwarf-rt_v6-rev0
+#      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
 
 build_script:
   - git describe --tags

bindings.go

@@ -2,6 +2,7 @@ package reindexer
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -555,6 +556,11 @@ func WithStrictJoinHandlers() interface{} {
 	return bindings.OptionStrictJoinHandlers{EnableStrictJoinHandlers: true}
 }
 
+// Enables connection to Reindexer using TLS. If tls.Config is nil TLS is disabled
+func WithTLSConfig(config *tls.Config) interface{} {
+	return bindings.OptionTLS{Config: config}
+}
+
 // WithReconnectionStrategy allows to configure the behavior during reconnect after error.
 // Strategy used for reconnect to server on connection error
 // AllowUnknownNodes allows to add dsn from cluster node, that was not set in client dsn list

bindings/builtinserver/config/config.go

@@ -84,6 +84,7 @@ func DefaultServerConfig() *ServerConfig {
 			HTTPThreading: "shared",
 			RPCAddr:       "0.0.0.0:6534",
 			RPCThreading:  "shared",
+			UnixRPCAddr:   "none",
 			Security:      false,
 		},
 		Logger: LoggerConf{

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v4.18.0"
+const ReindexerVersion = "v4.19.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (
@@ -200,5 +200,4 @@ const (
 const (
 	ShardingNotSet   = -1
 	ShardingProxyOff = -2
-	NotSharded       = -3
 )

bindings/cproto/connection.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -142,6 +143,8 @@ type connectionImpl struct {
 	requestDedicatedThread bool
 	loggerOwner            LoggerOwner
 	eventsHandler          bindings.EventsHandler
+
+	configTLS *tls.Config
 }
 
 type newConnParams struct {
@@ -154,6 +157,7 @@ type newConnParams struct {
 	requestDedicatedThread bool
 	caps                   bindings.BindingCapabilities
 	envetsHandler          bindings.EventsHandler
+	tls                    bindings.OptionTLS
 }
 
 func newConnection(
@@ -175,6 +179,7 @@ func newConnection(
 		requestDedicatedThread: params.requestDedicatedThread,
 		loggerOwner:            loggerOwner,
 		eventsHandler:          eventsHandler,
+		configTLS:              params.tls.Config,
 	}
 	for i := 0; i < queueSize; i++ {
 		c.seqs <- uint32(i)
@@ -259,15 +264,21 @@ func (c *connectionImpl) deadlineTicker() {
 }
 
 func (c *connectionImpl) connect(ctx context.Context, dsn *url.URL) (err error) {
-	var d net.Dialer
-	if dsn.Scheme == "cproto" {
-		if c.conn, err = d.DialContext(ctx, "tcp", dsn.Host); err != nil {
+	var netDialer net.Dialer
+	if dsn.Scheme == "cprotos" {
+		tlsDialer := tls.Dialer{Config: c.configTLS, NetDialer: &netDialer}
+		if c.conn, err = tlsDialer.DialContext(ctx, "tcp", dsn.Host); err != nil {
+			return err
+		}
+		c.conn.(*tls.Conn).NetConn().(*net.TCPConn).SetNoDelay(true)
+	} else if dsn.Scheme == "cproto" {
+		if c.conn, err = netDialer.DialContext(ctx, "tcp", dsn.Host); err != nil {
 			return err
 		}
 		c.conn.(*net.TCPConn).SetNoDelay(true)
 	} else {
-		d.LocalAddr = nil
-		if c.conn, err = d.DialContext(ctx, "unix", dsn.Host); err != nil {
+		netDialer.LocalAddr = nil
+		if c.conn, err = netDialer.DialContext(ctx, "unix", dsn.Host); err != nil {
 			return err
 		}
 	}

bindings/cproto/cproto.go

@@ -60,6 +60,7 @@ var emptyLogger bindings.NullLogger
 func init() {
 	rand.Seed(time.Now().UnixNano())
 	bindings.RegisterBinding("cproto", new(NetCProto))
+	bindings.RegisterBinding("cprotos", new(NetCProto))
 	if runtime.GOOS != "windows" {
 		bindings.RegisterBinding("ucproto", new(NetCProto))
 	}
@@ -88,6 +89,7 @@ type NetCProto struct {
 	lock               sync.RWMutex
 	logger             bindings.Logger
 	logMtx             sync.RWMutex
+	tls                bindings.OptionTLS
 }
 
 type dsn struct {
@@ -310,6 +312,8 @@ func (binding *NetCProto) Init(u []url.URL, eh bindings.EventsHandler, options .
 		case bindings.OptionReconnectionStrategy:
 			binding.dsn.reconnectionStrategy = v.Strategy
 			binding.dsn.allowUnknownNodes = v.AllowUnknownNodes
+		case bindings.OptionTLS:
+			binding.tls = v
 
 		default:
 			fmt.Printf("Unknown cproto option: %#v\n", option)
@@ -396,7 +400,8 @@ func (binding *NetCProto) createConnParams() newConnParams {
 		appName:                binding.appName,
 		enableCompression:      binding.compression.EnableCompression,
 		requestDedicatedThread: binding.dedicatedThreads.DedicatedThreads,
-		caps:                   binding.caps}
+		caps:                   binding.caps,
+		tls:                    binding.tls}
 }
 
 func (binding *NetCProto) createEventConn(ctx context.Context, connParams newConnParams, eventsSubOptsJSON []byte) error {

bindings/interface.go

@@ -2,6 +2,7 @@ package bindings
 
 import (
 	"context"
+	"crypto/tls"
 	"net/url"
 	"time"
 
@@ -384,6 +385,10 @@ type OptionReconnectionStrategy struct {
 	AllowUnknownNodes bool
 }
 
+type OptionTLS struct {
+	Config *tls.Config
+}
+
 type Status struct {
 	Err     error
 	CProto  StatusCProto

changelog.md

@@ -1,3 +1,32 @@
+# Version 4.19.0 (17.12.2024)
+
+## Core
+- [fea] Added automatic masking for reindexer user's credentials in log files and cluster/sharding JSON's (except for `async_replication` configs in `#config`-namespace)
+- [fix] Fixed assertion on attempt to use 'null'-values with `=`, `IN()`, `<`, `>`, `<=`, `>=` and `RANGE()` operators
+
+## Replication
+- [fea] Optimized memory footprint for online-updates queue
+
+## Reindexer server
+- [fea] Added `OpenSSL` support for HTTP-server (`https`) and RPC-server (`cprotos`). [Read more...](cpp_src/readme.md#tls-support)
+- [fea] Added special user roles `sharding` and `replication` with all required rights and restrictions for correspondig scenarios
+- [fea] Added support for SHA256/SHA512-based encription for user's passwords in `users.yml`. [Read more...](cpp_src/readme.md#authentication)
+- [fea] Added HTTP method `GET api/v1/user/role` to check current user's role
+- [fix] Fixed possible update buffer overflow for [events subscription](readme.md#events-subscription) in TCP-mode
+
+## Ported
+- [fea/fix] Ported all the fixes and features from [v3.30.0](https://github.com/Restream/reindexer/releases/tag/v3.30.0)
+
+## Face
+- [fea] Added `column_size` field to the Memory table
+- [fea] Added `splitter` field to the Full text index config
+- [fix] Fixed wrong info message about disabled statistics when the database didn't have Namespaces
+- [fix] Fixed last row position in the Memory table
+- [fix] Fixed wrong input field clearing on cache settings page
+- [fix] Fixed sort order icon on the tables
+- [fix] Fixed Replication Statistics table behavior on tabs switching
+- [fix] Fixed data saving in JSON view on the Fulltext index settings page
+
 # Version 4.18.0 (31.10.2024) *beta*
 ## Replication
 - [fix] Fixed possible origin LSN missmatch in snapshots during WAL/Force syncs

cpp_src/CMakeLists.txt

@@ -34,6 +34,11 @@ option(ENABLE_SSE "Enable SSE instructions" ON)
 option(ENABLE_SERVER_AS_PROCESS_IN_TEST "Run reindexer servers as separate processes in tests" OFF)
 option(ENABLE_V3_FOLLOWERS "Enable compatibility mode with reindexer v3 followers. This is temporary flag and will be removed in further releases" OFF)
 
+if(APPLE)
+  option(ENABLE_OPENSSL "Enable OpenSSL" OFF)
+else()
+  option(ENABLE_OPENSSL "Enable OpenSSL" ON)
+endif()
 
 if(NOT GRPC_PACKAGE_PROVIDER)
   set(GRPC_PACKAGE_PROVIDER "CONFIG")
@@ -45,7 +50,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "4.18.0")
+set (REINDEXER_VERSION_DEFAULT "4.19.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -308,6 +313,7 @@ add_library(${TARGET} STATIC ${HDRS} ${SRCS} ${VENDORS})
 add_definitions(-DREINDEX_CORE_BUILD=1)
 add_definitions(-DFMT_HEADER_ONLY=1)
 add_definitions(-DSPDLOG_FMT_EXTERNAL=1)
+add_definitions(-DFMT_USE_FULL_CACHE_DRAGONBOX=1)
 
 # add_definitions(-DREINDEX_FT_EXTRA_DEBUG=1)
 if (ENABLE_SERVER_AS_PROCESS_IN_TEST)
@@ -473,6 +479,13 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED ON)
 list(APPEND REINDEXER_LIBRARIES ${CMAKE_THREAD_LIBS_INIT} )
 
+include(CMakeRC)
+
+file(GLOB CHINA_DICT LIST_DIRECTORIES false ${REINDEXER_SOURCE_PATH}/resource/china_dict/*.lex)
+cmrc_add_resource_library(friso_dict_resources WHENCE ${REINDEXER_SOURCE_PATH}/resource ${CHINA_DICT})
+list(APPEND REINDEXER_LIBRARIES friso_dict_resources)
+add_dependencies(${TARGET} friso_dict_resources)
+
 if(WITH_CPPTRACE)
   ExternalProject_Add(
     cpptrace_lib
@@ -559,6 +572,18 @@ if(WIN32)
   list(APPEND REINDEXER_LIBRARIES shlwapi dbghelp ws2_32)
 endif()
 
+if (MSVC)
+    add_definitions(-DNOMINMAX)
+endif()
+
+if(ENABLE_OPENSSL)
+  find_package(OpenSSL)
+  if(OPENSSL_FOUND)
+    include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR})
+    add_definitions(-DWITH_OPENSSL)
+  endif()
+endif()
+
 set(REINDEXER_LIBRARIES_GLOBAL ${REINDEXER_LIBRARIES} PARENT_SCOPE)
 set(REINDEXER_LINK_DIRECTORIES_GLOBAL ${REINDEXER_LINK_DIRECTORIES} PARENT_SCOPE)
 
@@ -580,7 +605,6 @@ endif()
 if(NOT REINDEXER_VERSION_CUR_H STREQUAL REINDEXER_VERSION_H)
   file(WRITE ${PROJECT_BINARY_DIR}/reindexer_version.h ${REINDEXER_VERSION_H})
 endif()
-
 include_directories(${PROJECT_BINARY_DIR})
 
 string ( REGEX REPLACE "v([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)" "\\1.\\2.\\3" REINDEXER_VERSION ${REINDEXER_VERSION_FULL})

cpp_src/client/connectopts.h

@@ -5,10 +5,7 @@
 namespace reindexer {
 namespace client {
 
-enum ConnectOpt {
-	kConnectOptCreateIfMissing = 1 << 0,
-	kConnectOptCheckClusterID = 1 << 1,
-};
+enum ConnectOpt { kConnectOptCreateIfMissing = 1 << 0, kConnectOptCheckClusterID = 1 << 1 };
 
 struct ConnectOpts {
 	bool IsCreateDBIfMissing() const { return options & kConnectOptCreateIfMissing; }