DCOS-48854: Upgrade Elastic base-tech to 6.6.1. (#2976)

* Upgrade Elastic base-tech to 6.6.1.

* Run TLS tests with a trial license enabled.

And healthcheck credentials configured.

* Black formatting, assign constants with long names to variables.

This commit introduces no functional changes. It is an aesthetic refactoring.

* Setup Kibana credentials.

This fixes the TLS test, so we can remove the "skip" annotation.

* Fix test_crud_over_tls.

* Set Kibana's `server.rewriteBasePath: false` and fix health check.

* Make sure service name is there for tests to use it.

* Enable sdk_upgrade functions to take both 'from' and 'to' versions.

* Only run X-Pack related upgrade tests for ES5->ES6 upgrades.

frameworks/cassandra/tests/test_racks.py

@@ -54,5 +54,5 @@ def test_custom_rack_upgrade():
         config.PACKAGE_NAME,
         foldered_service_name,
         config.DEFAULT_TASK_COUNT,
-        additional_options=service_options,
+        from_options=service_options,
     )

frameworks/cassandra/tests/test_sanity.py

@@ -32,7 +32,7 @@ def configure_package(configure_security):
             config.PACKAGE_NAME,
             config.get_foldered_service_name(),
             config.DEFAULT_TASK_COUNT,
-            additional_options={"service": {"name": config.get_foldered_service_name()}},
+            from_options={"service": {"name": config.get_foldered_service_name()}},
         )
 
         yield  # let the test session execute

frameworks/elastic/tests/config.py

@@ -9,7 +9,6 @@
 import sdk_hosts
 import sdk_install
 import sdk_networks
-import sdk_repository
 import sdk_service
 import sdk_upgrade
 import sdk_utils
@@ -111,7 +110,9 @@ def check_custom_elasticsearch_cluster_setting(
     setting_path: Optional[str] = None,
     expected_value: Optional[str] = None,
 ) -> bool:
-    settings = _curl_query(service_name, "GET", "_cluster/settings?include_defaults=true")["defaults"]
+    settings = _curl_query(service_name, "GET", "_cluster/settings?include_defaults=true")[
+        "defaults"
+    ]
     if not settings:
         return False
     actual_value = get_in(setting_path, settings)
@@ -125,8 +126,7 @@ def check_custom_elasticsearch_cluster_setting(
     wait_fixed=1000, stop_max_delay=DEFAULT_TIMEOUT * 1000, retry_on_result=lambda res: not res
 )
 def wait_for_expected_nodes_to_exist(
-    service_name: str = SERVICE_NAME,
-    task_count: int = DEFAULT_TASK_COUNT,
+    service_name: str = SERVICE_NAME, task_count: int = DEFAULT_TASK_COUNT
 ) -> bool:
     result = _curl_query(service_name, "GET", "_cluster/health")
     if not result or "number_of_nodes" not in result:
@@ -140,10 +140,7 @@ def wait_for_expected_nodes_to_exist(
 @retrying.retry(
     wait_fixed=1000, stop_max_delay=DEFAULT_TIMEOUT * 1000, retry_on_result=lambda res: not res
 )
-def check_kibana_plugin_installed(
-    plugin_name: str,
-    service_name: str = SERVICE_NAME,
-) -> bool:
+def check_kibana_plugin_installed(plugin_name: str, service_name: str = SERVICE_NAME) -> bool:
     task_sandbox = sdk_cmd.get_task_sandbox_path(service_name)
     # Environment variables aren't available on DC/OS 1.9 so we manually inject MESOS_SANDBOX (and
     # can't use ELASTIC_VERSION).
@@ -163,7 +160,7 @@ def check_kibana_plugin_installed(
     wait_fixed=1000, stop_max_delay=DEFAULT_TIMEOUT * 1000, retry_on_result=lambda res: not res
 )
 def check_elasticsearch_plugin_installed(
-    plugin_name: str, service_name: str = SERVICE_NAME,
+    plugin_name: str, service_name: str = SERVICE_NAME
 ) -> bool:
     result = _get_hosts_with_plugin(service_name, plugin_name)
     return result is not None and len(result) == DEFAULT_TASK_COUNT
@@ -173,8 +170,7 @@ def check_elasticsearch_plugin_installed(
     wait_fixed=1000, stop_max_delay=DEFAULT_TIMEOUT * 1000, retry_on_result=lambda res: not res
 )
 def check_elasticsearch_plugin_uninstalled(
-    plugin_name: str,
-    service_name: str = SERVICE_NAME,
+    plugin_name: str, service_name: str = SERVICE_NAME
 ) -> bool:
     result = _get_hosts_with_plugin(service_name, plugin_name)
     return result is not None and result == []
@@ -236,9 +232,14 @@ def verify_commercial_api_status(
     http_user: Optional[str] = None,
     http_password: Optional[str] = None,
 ) -> bool:
-    return bool(verify_graph_explore_endpoint(
-        is_expected_to_be_enabled, service_name, http_user=http_user, http_password=http_password
-    ))
+    return bool(
+        verify_graph_explore_endpoint(
+            is_expected_to_be_enabled,
+            service_name,
+            http_user=http_user,
+            http_password=http_password,
+        )
+    )
 
 
 # On Elastic 6.x, the "Graph Explore API" is available when the Elasticsearch cluster is configured
@@ -330,17 +331,27 @@ def verify_xpack_license(
     wait_fixed=1000, stop_max_delay=5 * 1000, retry_on_result=lambda return_value: not return_value
 )
 def setup_passwords(
-    service_name: str = SERVICE_NAME,
-    task_name: str = "master-0-node",
+    service_name: str = SERVICE_NAME, task_name: str = "master-0-node", https: bool = False
 ) -> Union[bool, Dict[str, str]]:
+    if https:
+        master_0_node_dns = sdk_networks.get_endpoint(PACKAGE_NAME, service_name, "master-http")[
+            "dns"
+        ][0]
+        url = "--url https://{}".format(master_0_node_dns)
+    else:
+        url = ""
+
     cmd = "\n".join(
         [
             "set -x",
             "export JAVA_HOME=$(ls -d ${MESOS_SANDBOX}/jdk*/jre/)",
             "ELASTICSEARCH_PATH=$(ls -d ${MESOS_SANDBOX}/elasticsearch-*/)",
-            "${ELASTICSEARCH_PATH}/bin/elasticsearch-setup-passwords auto --batch --verbose",
+            "${{ELASTICSEARCH_PATH}}/bin/elasticsearch-setup-passwords auto --batch --verbose {}".format(
+                url
+            ),
         ]
     )
+
     full_cmd = "bash -c '{}'".format(cmd)
     _, stdout, _ = sdk_cmd.service_task_exec(service_name, task_name, full_cmd)
 
@@ -378,17 +389,16 @@ def explore_graph(
     return result
 
 
-def start_trial_license(
-    service_name: str = SERVICE_NAME,
-) -> Dict[str, Any]:
-    result = _curl_query(service_name, "POST", "_xpack/license/start_trial?acknowledge=true")
+def start_trial_license(service_name: str = SERVICE_NAME, https: bool = False) -> Dict[str, Any]:
+    result = _curl_query(
+        service_name, "POST", "_xpack/license/start_trial?acknowledge=true", https=https
+    )
     assert isinstance(result, dict)
     return result
 
 
 def get_elasticsearch_indices_stats(
-    index_name: str,
-    service_name: str = SERVICE_NAME,
+    index_name: str, service_name: str = SERVICE_NAME
 ) -> Dict[str, Any]:
     result = _curl_query(service_name, "GET", "{}/_stats".format(index_name))
     assert isinstance(result, dict)
@@ -502,16 +512,18 @@ def _curl_query(
 ) -> Optional[Union[str, Dict[str, Any]]]:
     protocol = "https" if https else "http"
 
-    if http_password and not http_user:
-        raise Exception(
-            "HTTP authentication won't work with just a password. Needs at least user, or both user AND password"
-        )
-
-    credentials = ""
-    if http_user:
-        credentials = "-u {}".format(http_user)
     if http_password:
-        credentials = "{}:{}".format(credentials, http_password)
+        if not http_user:
+            http_user = DEFAULT_ELASTICSEARCH_USER
+            log.info("Using default basic HTTP user: '%s'", http_user)
+
+        credentials = "-u {}:{}".format(http_user, http_password)
+    else:
+        if http_user:
+            raise Exception(
+                "HTTP authentication won't work with just a user. Needs both user AND password"
+            )
+        credentials = ""
 
     host = sdk_hosts.autoip_host(service_name, task, _master_zero_http_port(service_name))
 
@@ -546,18 +558,21 @@ def build_errmsg(msg: str) -> str:
         return None
 
 
-# TODO(mpereira): it is safe to remove this test after the 6.x release.
 def test_xpack_enabled_update(
     service_name: str,
     from_xpack_enabled: bool,
     to_xpack_enabled: bool,
+    from_version: str,
+    to_version: str = "stub-universe",
 ) -> None:
     sdk_upgrade.test_upgrade(
         PACKAGE_NAME,
         service_name,
         DEFAULT_TASK_COUNT,
-        additional_options={"elasticsearch": {"xpack_enabled": from_xpack_enabled}},
-        test_version_additional_options={
+        from_version=from_version,
+        from_options={"elasticsearch": {"xpack_enabled": from_xpack_enabled}},
+        to_version=to_version,
+        to_options={
             "service": {"update_strategy": "parallel"},
             "elasticsearch": {"xpack_enabled": to_xpack_enabled},
         },
@@ -566,25 +581,17 @@ def test_xpack_enabled_update(
     wait_for_expected_nodes_to_exist(service_name=service_name, task_count=DEFAULT_TASK_COUNT)
 
 
-# TODO(mpereira): change this to xpack_security_enabled to xpack_security_enabled after the 6.x
-# release.
-def test_update_from_xpack_enabled_to_xpack_security_enabled(
-    service_name: str,
-    xpack_enabled: bool,
-    xpack_security_enabled: bool,
+def test_xpack_security_enabled_update(
+    service_name: str, from_xpack_security_enabled: bool, to_xpack_security_enabled: bool
 ) -> None:
-    assert not (
-        xpack_enabled is True and xpack_security_enabled is True
-    ), "This function does not handle the 'xpack_enabled: True' to 'xpack_security_enabled: True' upgrade scenario"
-
     sdk_upgrade.test_upgrade(
         PACKAGE_NAME,
         service_name,
         DEFAULT_TASK_COUNT,
-        additional_options={"elasticsearch": {"xpack_enabled": xpack_enabled}},
-        test_version_additional_options={
+        from_options={"elasticsearch": {"xpack_security_enabled": from_xpack_security_enabled}},
+        to_options={
             "service": {"update_strategy": "parallel"},
-            "elasticsearch": {"xpack_security_enabled": xpack_security_enabled},
+            "elasticsearch": {"xpack_security_enabled": to_xpack_security_enabled},
         },
     )
 
@@ -596,6 +603,8 @@ def test_upgrade_from_xpack_enabled(
     service_name: str,
     options: Dict[str, Any],
     expected_task_count: int,
+    from_version: str,
+    to_version: str = "stub-universe",
 ) -> None:
     # This test needs to run some code in between the Universe version installation and the upgrade
     # to the 'stub-universe' version, so it cannot use `sdk_upgrade.test_upgrade`.
@@ -604,16 +613,12 @@ def test_upgrade_from_xpack_enabled(
 
     sdk_install.uninstall(package_name, service_name)
 
-    # Move Universe repo to the top of the repo list so that we can first install the Universe
-    # version.
-    _, universe_version = sdk_repository.move_universe_repo(package_name, universe_repo_index=0)
-
     sdk_install.install(
         package_name,
         service_name,
         expected_running_tasks=expected_task_count,
         additional_options={"elasticsearch": {"xpack_enabled": True}},
-        package_version=universe_version,
+        package_version=from_version,
     )
 
     document_es_5_id = 1
@@ -641,16 +646,12 @@ def test_upgrade_from_xpack_enabled(
         http_password=http_password,
     )
 
-    # Move Universe repo back to the bottom of the repo list so that we can upgrade to the version
-    # under test.
-    _, test_version = sdk_repository.move_universe_repo(package_name)
-
     # First we upgrade to "X-Pack security enabled" set to false on ES6, so that we can use the
     # X-Pack migration assistance and upgrade APIs.
     sdk_upgrade.update_or_upgrade_or_downgrade(
         package_name,
         service_name,
-        test_version,
+        to_version,
         {
             "service": {"update_strategy": "parallel"},
             "elasticsearch": {"xpack_security_enabled": False},