{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "TSG027 - Observe cluster deployment\n", "===================================\n", "\n", "Description\n", "-----------\n", "\n", "To troubleshoot SQL Server big data cluster create issues the following\n", "commands are often useful for pinpointing underlying causes.\n", "\n", "Steps\n", "-----\n", "\n", "### Parameters" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [ "parameters" ] }, "outputs": [], "source": [ "tail_lines = 1000" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Common functions\n", "\n", "Define helper functions used in this notebook." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [ "hide_input" ] }, "outputs": [], "source": [ "# Define `run` function for transient fault handling, suggestions on error, and scrolling updates on Windows\n", "import sys\n", "import os\n", "import re\n", "import json\n", "import platform\n", "import shlex\n", "import shutil\n", "import datetime\n", "\n", "from subprocess import Popen, PIPE\n", "from IPython.display import Markdown\n", "\n", "retry_hints = {}\n", "error_hints = {}\n", "install_hint = {}\n", "\n", "first_run = True\n", "rules = None\n", "\n", "def run(cmd, return_output=False, no_output=False, retry_count=0):\n", " \"\"\"\n", " Run shell command, stream stdout, print stderr and optionally return output\n", " \"\"\"\n", " MAX_RETRIES = 5\n", " output = \"\"\n", " retry = False\n", "\n", " global first_run\n", " global rules\n", "\n", " if first_run:\n", " first_run = False\n", " rules = load_rules()\n", "\n", " # shlex.split is required on bash and for Windows paths with spaces\n", " #\n", " cmd_actual = shlex.split(cmd)\n", "\n", " # Store this (i.e. kubectl, python etc.) to support binary context aware error_hints and retries\n", " #\n", " user_provided_exe_name = cmd_actual[0].lower()\n", "\n", " # When running python, use the python in the ADS sandbox ({sys.executable})\n", " #\n", " if cmd.startswith(\"python \"):\n", " cmd_actual[0] = cmd_actual[0].replace(\"python\", sys.executable)\n", "\n", " # On Mac, when ADS is not launched from terminal, LC_ALL may not be set, which causes pip installs to fail\n", " # with:\n", " #\n", " # UnicodeDecodeError: 'ascii' codec can't decode byte 0xc5 in position 4969: ordinal not in range(128)\n", " #\n", " # Setting it to a default value of \"en_US.UTF-8\" enables pip install to complete\n", " #\n", " if platform.system() == \"Darwin\" and \"LC_ALL\" not in os.environ:\n", " os.environ[\"LC_ALL\"] = \"en_US.UTF-8\"\n", "\n", " # To aid supportabilty, determine which binary file will actually be executed on the machine\n", " #\n", " which_binary = None\n", "\n", " # Special case for CURL on Windows. The version of CURL in Windows System32 does not work to\n", " # get JWT tokens, it returns \"(56) Failure when receiving data from the peer\". If another instance\n", " # of CURL exists on the machine use that one. (Unfortunately the curl.exe in System32 is almost\n", " # always the first curl.exe in the path, and it can't be uninstalled from System32, so here we\n", " # look for the 2nd installation of CURL in the path)\n", " if platform.system() == \"Windows\" and cmd.startswith(\"curl \"):\n", " path = os.getenv('PATH')\n", " for p in path.split(os.path.pathsep):\n", " p = os.path.join(p, \"curl.exe\")\n", " if os.path.exists(p) and os.access(p, os.X_OK):\n", " if p.lower().find(\"system32\") == -1:\n", " cmd_actual[0] = p\n", " which_binary = p\n", " break\n", "\n", " # Find the path based location (shutil.which) of the executable that will be run (and display it to aid supportability), this\n", " # seems to be required for .msi installs of azdata.cmd/az.cmd. (otherwise Popen returns FileNotFound) \n", " #\n", " # NOTE: Bash needs cmd to be the list of the space separated values hence shlex.split.\n", " #\n", " if which_binary == None:\n", " which_binary = shutil.which(cmd_actual[0])\n", "\n", " if which_binary == None:\n", " if user_provided_exe_name in install_hint and install_hint[user_provided_exe_name] is not None:\n", " display(Markdown(f'HINT: Use [{install_hint[user_provided_exe_name][0]}]({install_hint[user_provided_exe_name][1]}) to resolve this issue.'))\n", "\n", " raise FileNotFoundError(f\"Executable '{cmd_actual[0]}' not found in path (where/which)\")\n", " else: \n", " cmd_actual[0] = which_binary\n", "\n", " start_time = datetime.datetime.now().replace(microsecond=0)\n", "\n", " print(f\"START: {cmd} @ {start_time} ({datetime.datetime.utcnow().replace(microsecond=0)} UTC)\")\n", " print(f\" using: {which_binary} ({platform.system()} {platform.release()} on {platform.machine()})\")\n", " print(f\" cwd: {os.getcwd()}\")\n", "\n", " # Command-line tools such as CURL and AZDATA HDFS commands output\n", " # scrolling progress bars, which causes Jupyter to hang forever, to\n", " # workaround this, use no_output=True\n", " #\n", "\n", " # Work around a infinite hang when a notebook generates a non-zero return code, break out, and do not wait\n", " #\n", " wait = True \n", "\n", " try:\n", " if no_output:\n", " p = Popen(cmd_actual)\n", " else:\n", " p = Popen(cmd_actual, stdout=PIPE, stderr=PIPE, bufsize=1)\n", " with p.stdout:\n", " for line in iter(p.stdout.readline, b''):\n", " line = line.decode()\n", " if return_output:\n", " output = output + line\n", " else:\n", " if cmd.startswith(\"azdata notebook run\"): # Hyperlink the .ipynb file\n", " regex = re.compile(' \"(.*)\"\\: \"(.*)\"') \n", " match = regex.match(line)\n", " if match:\n", " if match.group(1).find(\"HTML\") != -1:\n", " display(Markdown(f' - \"{match.group(1)}\": \"{match.group(2)}\"'))\n", " else:\n", " display(Markdown(f' - \"{match.group(1)}\": \"[{match.group(2)}]({match.group(2)})\"'))\n", "\n", " wait = False\n", " break # otherwise infinite hang, have not worked out why yet.\n", " else:\n", " print(line, end='')\n", " if rules is not None:\n", " apply_expert_rules(line)\n", "\n", " if wait:\n", " p.wait()\n", " except FileNotFoundError as e:\n", " if install_hint is not None:\n", " display(Markdown(f'HINT: Use {install_hint} to resolve this issue.'))\n", "\n", " raise FileNotFoundError(f\"Executable '{cmd_actual[0]}' not found in path (where/which)\") from e\n", "\n", " exit_code_workaround = 0 # WORKAROUND: azdata hangs on exception from notebook on p.wait()\n", "\n", " if not no_output:\n", " for line in iter(p.stderr.readline, b''):\n", " line_decoded = line.decode()\n", "\n", " # azdata emits a single empty line to stderr when doing an hdfs cp, don't\n", " # print this empty \"ERR:\" as it confuses.\n", " #\n", " if line_decoded == \"\":\n", " continue\n", " \n", " print(f\"STDERR: {line_decoded}\", end='')\n", "\n", " if line_decoded.startswith(\"An exception has occurred\") or line_decoded.startswith(\"ERROR: An error occurred while executing the following cell\"):\n", " exit_code_workaround = 1\n", "\n", " if user_provided_exe_name in error_hints:\n", " for error_hint in error_hints[user_provided_exe_name]:\n", " if line_decoded.find(error_hint[0]) != -1:\n", " display(Markdown(f'HINT: Use [{error_hint[1]}]({error_hint[2]}) to resolve this issue.'))\n", "\n", " if rules is not None:\n", " apply_expert_rules(line_decoded)\n", "\n", " if user_provided_exe_name in retry_hints:\n", " for retry_hint in retry_hints[user_provided_exe_name]:\n", " if line_decoded.find(retry_hint) != -1:\n", " if retry_count < MAX_RETRIES:\n", " print(f\"RETRY: {retry_count} (due to: {retry_hint})\")\n", " retry_count = retry_count + 1\n", " output = run(cmd, return_output=return_output, retry_count=retry_count)\n", "\n", " if return_output:\n", " return output\n", " else:\n", " return\n", "\n", " elapsed = datetime.datetime.now().replace(microsecond=0) - start_time\n", "\n", " # WORKAROUND: We avoid infinite hang above in the `azdata notebook run` failure case, by inferring success (from stdout output), so\n", " # don't wait here, if success known above\n", " #\n", " if wait: \n", " if p.returncode != 0:\n", " raise SystemExit(f'Shell command:\\n\\n\\t{cmd} ({elapsed}s elapsed)\\n\\nreturned non-zero exit code: {str(p.returncode)}.\\n')\n", " else:\n", " if exit_code_workaround !=0 :\n", " raise SystemExit(f'Shell command:\\n\\n\\t{cmd} ({elapsed}s elapsed)\\n\\nreturned non-zero exit code: {str(exit_code_workaround)}.\\n')\n", "\n", "\n", " print(f'\\nSUCCESS: {elapsed}s elapsed.\\n')\n", "\n", " if return_output:\n", " return output\n", "\n", "def load_json(filename):\n", " with open(filename, encoding=\"utf8\") as json_file:\n", " return json.load(json_file)\n", "\n", "def load_rules():\n", "\n", " try:\n", "\n", " # Load this notebook as json to get access to the expert rules in the notebook metadata.\n", " #\n", " j = load_json(\"tsg027-observe-bdc-create.ipynb\")\n", "\n", " except:\n", " pass # If the user has renamed the book, we can't load ourself. NOTE: Is there a way in Jupyter, to know your own filename?\n", "\n", " else:\n", " if \"metadata\" in j and \\\n", " \"azdata\" in j[\"metadata\"] and \\\n", " \"expert\" in j[\"metadata\"][\"azdata\"] and \\\n", " \"rules\" in j[\"metadata\"][\"azdata\"][\"expert\"]:\n", "\n", " rules = j[\"metadata\"][\"azdata\"][\"expert\"][\"rules\"]\n", "\n", " rules.sort() # Sort rules, so they run in priority order (the [0] element). Lowest value first.\n", "\n", " # print (f\"EXPERT: There are {len(rules)} rules to evaluate.\")\n", "\n", " return rules\n", "\n", "def apply_expert_rules(line):\n", "\n", " global rules\n", "\n", " for rule in rules:\n", "\n", " # rules that have 9 elements are the injected (output) rules (the ones we want). Rules\n", " # with only 8 elements are the source (input) rules, which are not expanded (i.e. TSG029,\n", " # not ../repair/tsg029-nb-name.ipynb)\n", " if len(rule) == 9:\n", " notebook = rule[1]\n", " cell_type = rule[2]\n", " output_type = rule[3] # i.e. stream or error\n", " output_type_name = rule[4] # i.e. ename or name \n", " output_type_value = rule[5] # i.e. SystemExit or stdout\n", " details_name = rule[6] # i.e. evalue or text \n", " expression = rule[7].replace(\"\\\\*\", \"*\") # Something escaped *, and put a \\ in front of it!\n", "\n", " # print(f\"EXPERT: If rule '{expression}' satisfied', run '{notebook}'.\")\n", "\n", " if re.match(expression, line, re.DOTALL):\n", "\n", " # print(\"EXPERT: MATCH: name = value: '{0}' = '{1}' matched expression '{2}', therefore HINT '{4}'\".format(output_type_name, output_type_value, expression, notebook))\n", "\n", " match_found = True\n", "\n", " display(Markdown(f'HINT: Use [{notebook}]({notebook}) to resolve this issue.'))\n", "\n", "\n", "\n", "\n", "print('Common functions defined successfully.')\n", "\n", "# Hints for binary (transient fault) retry, (known) error and install guide\n", "#\n", "retry_hints = {'kubectl': ['A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond'], 'azdata': ['Endpoint sql-server-master does not exist', 'Endpoint livy does not exist', 'Failed to get state for cluster', 'Endpoint webhdfs does not exist', 'Adaptive Server is unavailable or does not exist', 'Error: Address already in use']}\n", "error_hints = {'kubectl': [['no such host', 'TSG010 - Get configuration contexts', '../monitor-k8s/tsg010-get-kubernetes-contexts.ipynb'], ['no such host', 'TSG011 - Restart sparkhistory server', '../repair/tsg011-restart-sparkhistory-server.ipynb'], ['No connection could be made because the target machine actively refused it', 'TSG056 - Kubectl fails with No connection could be made because the target machine actively refused it', '../repair/tsg056-kubectl-no-connection-could-be-made.ipynb']], 'azdata': [['azdata login', 'SOP028 - azdata login', '../common/sop028-azdata-login.ipynb'], ['The token is expired', 'SOP028 - azdata login', '../common/sop028-azdata-login.ipynb'], ['Reason: Unauthorized', 'SOP028 - azdata login', '../common/sop028-azdata-login.ipynb'], ['Max retries exceeded with url: /api/v1/bdc/endpoints', 'SOP028 - azdata login', '../common/sop028-azdata-login.ipynb'], ['Look at the controller logs for more details', 'TSG027 - Observe cluster deployment', '../diagnose/tsg027-observe-bdc-create.ipynb'], ['provided port is already allocated', 'TSG062 - Get tail of all previous container logs for pods in BDC namespace', '../log-files/tsg062-tail-bdc-previous-container-logs.ipynb'], ['Create cluster failed since the existing namespace', 'SOP061 - Delete a big data cluster', '../install/sop061-delete-bdc.ipynb'], ['Failed to complete kube config setup', 'TSG067 - Failed to complete kube config setup', '../repair/tsg067-failed-to-complete-kube-config-setup.ipynb'], ['Error processing command: \"ApiError', 'TSG110 - Azdata returns ApiError', '../repair/tsg110-azdata-returns-apierror.ipynb'], ['Error processing command: \"ControllerError', 'TSG036 - Controller logs', '../log-analyzers/tsg036-get-controller-logs.ipynb'], ['ERROR: 500', 'TSG046 - Knox gateway logs', '../log-analyzers/tsg046-get-knox-logs.ipynb'], ['Data source name not found and no default driver specified', 'SOP069 - Install ODBC for SQL Server', '../install/sop069-install-odbc-driver-for-sql-server.ipynb'], [\"Can't open lib 'ODBC Driver 17 for SQL Server\", 'SOP069 - Install ODBC for SQL Server', '../install/sop069-install-odbc-driver-for-sql-server.ipynb'], ['Control plane upgrade failed. Failed to upgrade controller.', 'TSG108 - View the controller upgrade config map', '../diagnose/tsg108-controller-failed-to-upgrade.ipynb']]}\n", "install_hint = {'kubectl': ['SOP036 - Install kubectl command line interface', '../install/sop036-install-kubectl.ipynb'], 'azdata': ['SOP055 - Install azdata command line interface', '../install/sop055-install-azdata.ipynb']}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Show the Kubernetes version information" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('kubectl version -o yaml')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Show the AZDATA version information" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('azdata --version')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Show the Kubernetes nodes" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('kubectl get nodes')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Show the Kubernetes namespaces\n", "\n", "See the namespace for the new cluster, it should be displayed in the\n", "list of Kubernetes namespaces. Creating the namespace is one of the\n", "first actions performed by \u2018azdata cluster create\u2019" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('kubectl get namespace')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get the Kubernetes namespace for the big data cluster\n", "\n", "Get the namespace of the big data cluster use the kubectl command line\n", "interface .\n", "\n", "NOTE: If there is more than one big data cluster in the target\n", "Kubernetes cluster, then set \\[0\\] to the correct value for the big data\n", "cluster." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [ "hide_input" ] }, "outputs": [], "source": [ "# Place Kubernetes namespace name for BDC into 'namespace' variable\n", "\n", "try:\n", " namespace = run(f'kubectl get namespace --selector=MSSQL_CLUSTER -o jsonpath={{.items[0].metadata.name}}', return_output=True)\n", "except:\n", " from IPython.display import Markdown\n", " print(f\"ERROR: Unable to find a Kubernetes namespace with label 'MSSQL_CLUSTER'. SQL Server Big Data Cluster Kubernetes namespaces contain the label 'MSSQL_CLUSTER'.\")\n", " display(Markdown(f'HINT: Use [TSG081 - Get namespaces (Kubernetes)](../monitor-k8s/tsg081-get-kubernetes-namespaces.ipynb) to resolve this issue.'))\n", " display(Markdown(f'HINT: Use [TSG010 - Get configuration contexts](../monitor-k8s/tsg010-get-kubernetes-contexts.ipynb) to resolve this issue.'))\n", " display(Markdown(f'HINT: Use [SOP011 - Set kubernetes configuration context](../common/sop011-set-kubernetes-context.ipynb) to resolve this issue.'))\n", " raise\n", "else:\n", " print(f'The SQL Server Big Data Cluster Kubernetes namespace is: {namespace}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Show the System pods for the big data cluster\n", "\n", "Show the Kubernetes system pods" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('kubectl get pods -n kube-system -o wide')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Show the Kubernetes pods for the big data cluster\n", "\n", "Show the SQL Server big data cluster pods" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run(f'kubectl get pods -n {namespace} -o wide')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Show the Kubernetes services for the big data cluster\n", "\n", "Show the SQL Server big data cluster services" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run(f'kubectl get service -n {namespace}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Show the Kubernetes events for the big data cluster\n", "\n", "Show the SQL Server big data cluster events" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run(f'kubectl get events -n {namespace}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Describe the `controller` pod\n", "\n", "Describe the `controller` pod" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "controller = run(f'kubectl get pod --selector=app=controller -n {namespace} -o jsonpath={{.items[0].metadata.name}}', return_output=True)\n", "\n", "run(f'kubectl describe pod/{controller} -n {namespace}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get the `controller` container logs\n", "\n", "Get the `controller` container logs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run(f'kubectl logs pod/{controller} -c controller -n {namespace} --tail={tail_lines}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get the `controller` database `mssql-server` container logs\n", "\n", "Get the `controller` database `mssql-server` container logs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run(f'kubectl logs pod/controldb-0 -c mssql-server -n {namespace} --tail={tail_lines}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get the `controller` `security-support` container logs\n", "\n", "Get the `controller` `security-support` container logs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run(f'kubectl logs pod/{controller} -c security-support -n {namespace} --tail={tail_lines}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get the `controller` `fluentbit` container logs\n", "\n", "Get the `controller` `fluentbit` container logs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run(f'kubectl logs pod/{controller} -c fluentbit -n {namespace} --tail={tail_lines}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Describe the `controller watchdog` pod\n", "\n", "Describe the `controller watchdog` pod" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "controlwd = None\n", "\n", "try:\n", " controlwd = run(f'kubectl get pod --selector=app=controlwd -n {namespace} -o jsonpath={{.items[0].metadata.name}}', return_output=True)\n", " run(f'kubectl describe pod/{controlwd} -n {namespace}')\n", "except:\n", " print(\"Skipping 'controller watchdog', it has not been created yet\" )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get the `controller watchdog` container logs\n", "\n", "Get the `controller watchdog` container logs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "if controlwd is not None:\n", " run(f'kubectl logs pod/{controlwd} -n {namespace} --tail={tail_lines}')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print('Notebook execution complete.')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Related\n", "-------\n", "\n", "- [TSG001 - Run azdata copy-logs](../log-files/tsg001-copy-logs.ipynb)\n", "\n", "- [TSG002 -\n", " CrashLoopBackoff](../diagnose/tsg002-crash-loop-backoff.ipynb)" ] } ], "nbformat": 4, "nbformat_minor": 5, "metadata": { "kernelspec": { "name": "python3", "display_name": "Python 3" }, "azdata": { "side_effects": false } } }