{
"cells": [
{
"cell_type": "markdown",
"id": "ddc263cb",
"metadata": {},
"source": [
"### Reference Documents:\n",
"SparkMagic \n",
"Spark Doc"
]
},
{
"cell_type": "markdown",
"id": "49ebfe44",
"metadata": {},
"source": [
"# Installing Libraries within SYNTASA Notebooks (AMAZON AWS environments)\n",
"### When using SYNTASA you can either use the local notebook (which won't have apache spark available), this is useful for running simple Python/Scala Code.\n",
"### Additionally, users can attach a Runtime to run jobs on a bigger external cluster (for Spark and Distributed workloads). See the Gear icon to the right of this Notebook $\\rightarrow$\n"
]
},
{
"cell_type": "markdown",
"id": "8fa25902",
"metadata": {},
"source": [
"#### Installing Libraries using Jupyter Magic Commands. This is only for installing in the local environment (within the notebook) and can help when writing generic Python code without any Spark Dependencies"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "8053068a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mWARNING: The directory '/home/jovyan/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.\u001b[0m\u001b[33m\n",
"\u001b[0mRequirement already satisfied: pip in /opt/conda/lib/python3.7/site-packages (22.1.2)\n",
"Collecting pip\n",
" Downloading pip-22.2.2-py3-none-any.whl (2.0 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m127.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hInstalling collected packages: pip\n",
" Attempting uninstall: pip\n",
" Found existing installation: pip 22.1.2\n",
" Uninstalling pip-22.1.2:\n",
" Successfully uninstalled pip-22.1.2\n",
"Successfully installed pip-22.2.2\n",
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: The directory '/home/jovyan/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.\u001b[0m\u001b[33m\n",
"\u001b[0mRequirement already satisfied: boto3 in /opt/conda/lib/python3.7/site-packages (1.24.69)\n",
"Requirement already satisfied: pillow in /opt/conda/lib/python3.7/site-packages (9.2.0)\n",
"Requirement already satisfied: s3fs in /opt/conda/lib/python3.7/site-packages (0.4.2)\n",
"Collecting s3fs\n",
" Downloading s3fs-2022.8.2-py3-none-any.whl (27 kB)\n",
"Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/conda/lib/python3.7/site-packages (from boto3) (1.0.1)\n",
"Requirement already satisfied: botocore<1.28.0,>=1.27.69 in /opt/conda/lib/python3.7/site-packages (from boto3) (1.27.69)\n",
"Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from boto3) (0.6.0)\n",
"Collecting aiohttp!=4.0.0a0,!=4.0.0a1\n",
" Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m116.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting aiobotocore~=2.4.0\n",
" Downloading aiobotocore-2.4.0-py3-none-any.whl (65 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.8/65.8 kB\u001b[0m \u001b[31m140.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting fsspec==2022.8.2\n",
" Downloading fsspec-2022.8.2-py3-none-any.whl (140 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.8/140.8 kB\u001b[0m \u001b[31m222.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting aioitertools>=0.5.1\n",
" Downloading aioitertools-0.10.0-py3-none-any.whl (23 kB)\n",
"Collecting wrapt>=1.10.10\n",
" Downloading wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (75 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.2/75.2 kB\u001b[0m \u001b[31m177.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hINFO: pip is looking at multiple versions of fsspec to determine which version is compatible with other requirements. This could take a while.\n",
"INFO: pip is looking at multiple versions of to determine which version is compatible with other requirements. This could take a while.\n",
"INFO: pip is looking at multiple versions of s3fs to determine which version is compatible with other requirements. This could take a while.\n",
"Collecting s3fs\n",
" Downloading s3fs-2022.8.1-py3-none-any.whl (27 kB)\n",
"Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/conda/lib/python3.7/site-packages (from botocore<1.28.0,>=1.27.69->boto3) (2.8.1)\n",
"Requirement already satisfied: urllib3<1.27,>=1.25.4 in /opt/conda/lib/python3.7/site-packages (from botocore<1.28.0,>=1.27.69->boto3) (1.25.9)\n",
"Collecting fsspec==2022.8.1\n",
" Downloading fsspec-2022.8.1-py3-none-any.whl (140 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.8/140.8 kB\u001b[0m \u001b[31m224.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2022.8.0-py3-none-any.whl (27 kB)\n",
"Collecting fsspec==2022.8.0\n",
" Downloading fsspec-2022.8.0-py3-none-any.whl (140 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.0/141.0 kB\u001b[0m \u001b[31m215.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2022.7.1-py3-none-any.whl (27 kB)\n",
"Collecting aiobotocore~=2.3.4\n",
" Downloading aiobotocore-2.3.4-py3-none-any.whl (64 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.7/64.7 kB\u001b[0m \u001b[31m172.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting fsspec==2022.7.1\n",
" Downloading fsspec-2022.7.1-py3-none-any.whl (141 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.2/141.2 kB\u001b[0m \u001b[31m209.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2022.7.0-py3-none-any.whl (27 kB)\n",
"Collecting fsspec==2022.7.0\n",
" Downloading fsspec-2022.7.0-py3-none-any.whl (141 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.2/141.2 kB\u001b[0m \u001b[31m150.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2022.5.0-py3-none-any.whl (27 kB)\n",
"Collecting fsspec==2022.5.0\n",
" Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 kB\u001b[0m \u001b[31m191.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting aiobotocore~=2.3.0\n",
" Downloading aiobotocore-2.3.3.tar.gz (65 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.7/65.7 kB\u001b[0m \u001b[31m181.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-2.3.2.tar.gz (104 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.8/104.8 kB\u001b[0m \u001b[31m196.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-2.3.1.tar.gz (65 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.3/65.3 kB\u001b[0m \u001b[31m173.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-2.3.0.tar.gz (65 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.1/65.1 kB\u001b[0m \u001b[31m178.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2022.3.0-py3-none-any.whl (26 kB)\n",
"Collecting fsspec==2022.3.0\n",
" Downloading fsspec-2022.3.0-py3-none-any.whl (136 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m136.1/136.1 kB\u001b[0m \u001b[31m210.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting aiobotocore~=2.2.0\n",
" Downloading aiobotocore-2.2.0.tar.gz (59 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.7/59.7 kB\u001b[0m \u001b[31m185.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2022.2.0-py3-none-any.whl (26 kB)\n",
"Collecting fsspec==2022.02.0\n",
" Downloading fsspec-2022.2.0-py3-none-any.whl (134 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.9/134.9 kB\u001b[0m \u001b[31m127.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting aiobotocore~=2.1.0\n",
" Downloading aiobotocore-2.1.2.tar.gz (58 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.7/58.7 kB\u001b[0m \u001b[31m175.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-2.1.1.tar.gz (57 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m173.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-2.1.0.tar.gz (54 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 kB\u001b[0m \u001b[31m171.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hINFO: pip is looking at multiple versions of fsspec to determine which version is compatible with other requirements. This could take a while.\n",
"INFO: pip is looking at multiple versions of to determine which version is compatible with other requirements. This could take a while.\n",
"INFO: pip is looking at multiple versions of s3fs to determine which version is compatible with other requirements. This could take a while.\n",
"Collecting s3fs\n",
" Downloading s3fs-2022.1.0-py3-none-any.whl (25 kB)\n",
"Collecting fsspec==2022.01.0\n",
" Downloading fsspec-2022.1.0-py3-none-any.whl (133 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m133.2/133.2 kB\u001b[0m \u001b[31m205.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.11.1-py3-none-any.whl (25 kB)\n",
"Collecting aiobotocore~=2.0.1\n",
" Downloading aiobotocore-2.0.1.tar.gz (54 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.5/54.5 kB\u001b[0m \u001b[31m181.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hCollecting fsspec==2021.11.1\n",
" Downloading fsspec-2021.11.1-py3-none-any.whl (132 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m133.0/133.0 kB\u001b[0m \u001b[31m204.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.11.0-py3-none-any.whl (25 kB)\n",
"Collecting aiobotocore~=1.4.1\n",
" Downloading aiobotocore-1.4.2.tar.gz (52 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m158.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hCollecting fsspec==2021.11.0\n",
" Downloading fsspec-2021.11.0-py3-none-any.whl (132 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.4/132.4 kB\u001b[0m \u001b[31m213.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting aiobotocore~=1.4.1\n",
" Downloading aiobotocore-1.4.1.tar.gz (52 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.3/52.3 kB\u001b[0m \u001b[31m165.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.10.1-py3-none-any.whl (26 kB)\n",
"Collecting fsspec==2021.10.1\n",
" Downloading fsspec-2021.10.1-py3-none-any.whl (125 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m125.6/125.6 kB\u001b[0m \u001b[31m198.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.10.0-py3-none-any.whl (26 kB)\n",
"Collecting fsspec==2021.10.0\n",
" Downloading fsspec-2021.10.0-py3-none-any.whl (125 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m125.0/125.0 kB\u001b[0m \u001b[31m224.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hINFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
"INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
"INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
"Collecting s3fs\n",
" Downloading s3fs-2021.9.0-py3-none-any.whl (26 kB)\n",
"Collecting fsspec==2021.09.0\n",
" Downloading fsspec-2021.9.0-py3-none-any.whl (123 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.6/123.6 kB\u001b[0m \u001b[31m191.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.8.1-py3-none-any.whl (26 kB)\n",
"Collecting fsspec==2021.08.1\n",
" Downloading fsspec-2021.8.1-py3-none-any.whl (119 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.3/119.3 kB\u001b[0m \u001b[31m204.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting aiobotocore~=1.4.0\n",
" Downloading aiobotocore-1.4.0.tar.gz (51 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.6/51.6 kB\u001b[0m \u001b[31m160.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.8.0-py3-none-any.whl (26 kB)\n",
"Collecting fsspec==2021.07.0\n",
" Downloading fsspec-2021.7.0-py3-none-any.whl (118 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.1/118.1 kB\u001b[0m \u001b[31m211.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.7.0-py3-none-any.whl (25 kB)\n",
"Collecting aiobotocore>=1.0.1\n",
" Downloading aiobotocore-2.0.0.tar.gz (52 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.0/53.0 kB\u001b[0m \u001b[31m165.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-1.3.3.tar.gz (50 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.6/50.6 kB\u001b[0m \u001b[31m168.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-1.3.2.tar.gz (49 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m167.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-1.3.1.tar.gz (48 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.8/48.8 kB\u001b[0m \u001b[31m156.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-1.3.0.tar.gz (48 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.2/48.2 kB\u001b[0m \u001b[31m151.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-1.2.2.tar.gz (48 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.1/48.1 kB\u001b[0m \u001b[31m164.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-1.2.1.tar.gz (48 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.0/48.0 kB\u001b[0m \u001b[31m160.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-1.2.0.tar.gz (47 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.3/47.3 kB\u001b[0m \u001b[31m169.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Downloading aiobotocore-1.1.2-py3-none-any.whl (45 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.1/45.1 kB\u001b[0m \u001b[31m148.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.1.1-py3-none-any.whl (45 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.0/45.0 kB\u001b[0m \u001b[31m154.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.1.0-py3-none-any.whl (43 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m152.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.0.7-py3-none-any.whl (42 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.9/42.9 kB\u001b[0m \u001b[31m148.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.0.6-py3-none-any.whl (42 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.1/42.1 kB\u001b[0m \u001b[31m141.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.0.5-py3-none-any.whl (42 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.1/42.1 kB\u001b[0m \u001b[31m125.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.0.4-py3-none-any.whl (41 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 kB\u001b[0m \u001b[31m142.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.0.3-py3-none-any.whl (40 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 kB\u001b[0m \u001b[31m132.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.0.2-py3-none-any.whl (40 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 kB\u001b[0m \u001b[31m162.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Downloading aiobotocore-1.0.1-py3-none-any.whl (40 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.7/40.7 kB\u001b[0m \u001b[31m166.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.6.1-py3-none-any.whl (25 kB)\n",
"Collecting fsspec==2021.06.1\n",
" Downloading fsspec-2021.6.1-py3-none-any.whl (115 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.1/115.1 kB\u001b[0m \u001b[31m195.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.6.0-py3-none-any.whl (24 kB)\n",
"Collecting fsspec==2021.06.0\n",
" Downloading fsspec-2021.6.0-py3-none-any.whl (114 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.7/114.7 kB\u001b[0m \u001b[31m208.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.5.0-py3-none-any.whl (24 kB)\n",
"Collecting fsspec==2021.05.0\n",
" Downloading fsspec-2021.5.0-py3-none-any.whl (111 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.7/111.7 kB\u001b[0m \u001b[31m214.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-2021.4.0-py3-none-any.whl (23 kB)\n",
"Collecting fsspec==2021.04.0\n",
" Downloading fsspec-2021.4.0-py3-none-any.whl (108 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.3/108.3 kB\u001b[0m \u001b[31m203.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting s3fs\n",
" Downloading s3fs-0.6.0-py3-none-any.whl (23 kB)\n",
" Downloading s3fs-0.5.2-py3-none-any.whl (22 kB)\n",
" Downloading s3fs-0.5.1-py3-none-any.whl (21 kB)\n",
" Downloading s3fs-0.5.0-py3-none-any.whl (21 kB)\n",
"Requirement already satisfied: fsspec>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from s3fs) (0.7.4)\n",
"Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.69->boto3) (1.15.0)\n",
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m"
]
}
],
"source": [
"!pip3 install pip --upgrade\n",
"!pip3 install boto3 pillow s3fs --upgrade\n",
"\n",
"## Using the '!' symbol will execute the command within the local notebook environment\n",
"## You can run multiple commands in sequence"
]
},
{
"cell_type": "markdown",
"id": "753d47e0",
"metadata": {},
"source": [
"#### Once you run the above cell, you should be able to import the libraries you tried to install, in this case we installed \"boto3\", \"pillow\", \"s3fs\""
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "46421afd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Boto3 Version :: 1.24.24\n",
"Pillow Version :: 9.2.0\n",
"S3FS Version :: 0.4.2\n"
]
}
],
"source": [
"import boto3\n",
"import PIL\n",
"import s3fs\n",
"\n",
"print(f'Boto3 Version :: {boto3.__version__}')\n",
"print(f'Pillow Version :: {PIL.__version__}')\n",
"print(f'S3FS Version :: {s3fs.__version__}')"
]
},
{
"cell_type": "markdown",
"id": "fe14269b",
"metadata": {},
"source": [
"# Installing Libraries on an attached Runtime (Cluster) using Spark Magic and Syntasa Helper Methods\n",
"### In this next section, we will install libraries on a remote cluster which will be attached to this notebook using The Gear Icon on the Right $\\rightarrow$\n",
"### First click the gear icon on the right, Toggle the \"Connect Notebook to Syntasa Runtime\" option, and pick a Runtime and Notebook Instance (create a new one if an existing one isn't there)\n",
"### After creating a notebook instance, please click the Play button to start a Syntasa Runtime. Depending on the Runtime Size and AWS connection, this could take anywhere between 5-15 minutes\n",
"### Please note, after the runtime is spun up the Status in the top bar will change from [[ RUNTIME STATUS: Creating ]] to [[ RUNTIME STATUS: Running ]]. \n",
"#### At this point livy should connect and allow you to run code remotely\n",
"#### note that the cells after this point are annotated with \"%%spark\" at the top of each cell. These will run the code blocks on the remote cluster."
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "20c1390a",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Installing packages using AmazonClusterPyPackageInstaller\n",
"3"
]
}
],
"source": [
"%%spark\n",
"\n",
"from syn_utils import *\n",
"synutils.lib.installPyPI('boto3 pillow s3fs')"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "76bec6ac",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Boto3 Version :: 1.24.69\n",
"Pillow Version :: 9.2.0\n",
"S3FS Version :: 0.4.2"
]
}
],
"source": [
"%%spark \n",
"\n",
"import boto3\n",
"import PIL\n",
"import s3fs\n",
"\n",
"print(f'Boto3 Version :: {boto3.__version__}')\n",
"print(f'Pillow Version :: {PIL.__version__}')\n",
"print(f'S3FS Version :: {s3fs.__version__}')"
]
},
{
"cell_type": "markdown",
"id": "1155ceec",
"metadata": {},
"source": [
"# Additionally to Install libraries and be able to view the output logs of the install (to debug any issues)\n",
"### Please use the below work around to view debug logs in case you are running into issues using the previous method.\n",
"### This method additionally will let you run yum commands to install system libraries as well"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "e48e699f",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%spark\n",
"\n",
"from urllib3.exceptions import InsecureRequestWarning\n",
"from urllib3 import disable_warnings\n",
"from syn_utils.syn_notebook.lib import AmazonClusterPyPackageInstaller\n",
"import requests\n",
"\n",
"requests.packages.urllib3.disable_warnings()\n",
"disable_warnings(InsecureRequestWarning)\n",
"\n",
"TYPE_YUM='yum'\n",
"TYPE_PIP='pip'\n",
"INSTALL='install'\n",
"UNINSTALL='uninstall'\n",
"\n",
"def install_package(cmd):\n",
" amazon_installer = AmazonClusterPyPackageInstaller()\n",
" for ip in amazon_installer.node_ip_addresses:\n",
" amazon_installer.execute_command(cmd, ip)\n",
" \n",
"def custom_pkg_install(packages_arr, run_type, inst_uninst):\n",
" pkgs = ' '.join(packages_arr)\n",
" \n",
" cmd_to_run = ''\n",
" if run_type == TYPE_YUM:\n",
" cmd_to_run = f'sudo yum {inst_uninst} {pkgs} -y'\n",
" elif run_type == TYPE_PIP:\n",
" if inst_uninst == INSTALL:\n",
" cmd_to_run = f'sudo python3 -m pip {inst_uninst} {pkgs}'\n",
" elif inst_uninst == UNINSTALL:\n",
" cmd_to_run = f'sudo python3 -m pip {inst_uninst} {pkgs} -y' #need to add '-y' here as uninstall stops execution and waits for user input.\n",
" \n",
" print(f'Running Command [{cmd_to_run}]')\n",
" install_package(cmd_to_run)"
]
},
{
"cell_type": "markdown",
"id": "0e393a02",
"metadata": {},
"source": [
"#### Install a package from PIP using the above method (on an attached cluster/runtime)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "de9ed6b0",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Running Command [sudo python3 -m pip install requests matplotlib]\n",
"executing command on 10.0.1.220\n",
"Requirement already satisfied: requests in /usr/local/lib/python3.7/site-packages (2.28.1)\n",
"Collecting matplotlib\n",
"Downloading matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (11.2 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11.2/11.2 MB 65.0 MB/s eta 0:00:00\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/site-packages (from requests) (3.3)\n",
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/site-packages (from requests) (1.26.12)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/site-packages (from requests) (2022.6.15.1)\n",
"Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/site-packages (from requests) (2.1.1)\n",
"Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib64/python3.7/site-packages (from matplotlib) (9.2.0)\n",
"Collecting numpy>=1.17\n",
"Downloading numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 15.7/15.7 MB 58.4 MB/s eta 0:00:00\n",
"Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.7/site-packages (from matplotlib) (3.0.9)\n",
"Collecting packaging>=20.0\n",
"Downloading packaging-21.3-py3-none-any.whl (40 kB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 40.8/40.8 kB 10.3 MB/s eta 0:00:00\n",
"Collecting kiwisolver>=1.0.1\n",
"Downloading kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.1 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.1/1.1 MB 56.7 MB/s eta 0:00:00\n",
"Collecting fonttools>=4.22.0\n",
"Downloading fonttools-4.37.1-py3-none-any.whl (957 kB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 957.2/957.2 kB 60.8 MB/s eta 0:00:00\n",
"Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.7/site-packages (from matplotlib) (2.8.2)\n",
"Collecting cycler>=0.10\n",
"Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n",
"Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib) (4.3.0)\n",
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib) (1.13.0)\n",
"Installing collected packages: packaging, numpy, kiwisolver, fonttools, cycler, matplotlib\n",
"Successfully installed cycler-0.11.0 fonttools-4.37.1 kiwisolver-1.4.4 matplotlib-3.5.3 numpy-1.21.6 packaging-21.3\n",
"executing command on 10.0.1.201\n",
"Requirement already satisfied: requests in /usr/local/lib/python3.7/site-packages (2.28.1)\n",
"Collecting matplotlib\n",
"Downloading matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (11.2 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11.2/11.2 MB 42.4 MB/s eta 0:00:00\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/site-packages (from requests) (3.3)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/site-packages (from requests) (2022.6.15.1)\n",
"Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/site-packages (from requests) (2.1.1)\n",
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/site-packages (from requests) (1.26.12)\n",
"Collecting cycler>=0.10\n",
"Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n",
"Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib64/python3.7/site-packages (from matplotlib) (9.2.0)\n",
"Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.7/site-packages (from matplotlib) (2.8.2)\n",
"Collecting kiwisolver>=1.0.1\n",
"Downloading kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.1 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.1/1.1 MB 59.5 MB/s eta 0:00:00\n",
"Collecting fonttools>=4.22.0\n",
"Downloading fonttools-4.37.1-py3-none-any.whl (957 kB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 957.2/957.2 kB 63.3 MB/s eta 0:00:00\n",
"Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.7/site-packages (from matplotlib) (3.0.9)\n",
"Collecting numpy>=1.17\n",
"Downloading numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 15.7/15.7 MB 66.4 MB/s eta 0:00:00\n",
"Collecting packaging>=20.0\n",
"Downloading packaging-21.3-py3-none-any.whl (40 kB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 40.8/40.8 kB 8.4 MB/s eta 0:00:00\n",
"Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib) (4.3.0)\n",
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib) (1.13.0)\n",
"Installing collected packages: packaging, numpy, kiwisolver, fonttools, cycler, matplotlib\n",
"Successfully installed cycler-0.11.0 fonttools-4.37.1 kiwisolver-1.4.4 matplotlib-3.5.3 numpy-1.21.6 packaging-21.3\n",
"executing command on 10.0.1.119\n",
"Requirement already satisfied: requests in /usr/local/lib/python3.7/site-packages (2.28.1)\n",
"Collecting matplotlib\n",
"Downloading matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (11.2 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11.2/11.2 MB 39.5 MB/s eta 0:00:00\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/site-packages (from requests) (2022.6.15.1)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/site-packages (from requests) (3.3)\n",
"Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/site-packages (from requests) (2.1.1)\n",
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/site-packages (from requests) (1.26.12)\n",
"Collecting cycler>=0.10\n",
"Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n",
"Collecting packaging>=20.0\n",
"Downloading packaging-21.3-py3-none-any.whl (40 kB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 40.8/40.8 kB 7.5 MB/s eta 0:00:00\n",
"Collecting numpy>=1.17\n",
"Downloading numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 15.7/15.7 MB 62.1 MB/s eta 0:00:00\n",
"Collecting fonttools>=4.22.0\n",
"Downloading fonttools-4.37.1-py3-none-any.whl (957 kB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 957.2/957.2 kB 55.2 MB/s eta 0:00:00\n",
"Collecting kiwisolver>=1.0.1\n",
"Downloading kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.1 MB)\n",
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.1/1.1 MB 55.7 MB/s eta 0:00:00\n",
"Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.7/site-packages (from matplotlib) (2.8.2)\n",
"Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.7/site-packages (from matplotlib) (3.0.9)\n",
"Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib64/python3.7/site-packages (from matplotlib) (9.2.0)\n",
"Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib) (4.3.0)\n",
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib) (1.13.0)\n",
"Installing collected packages: packaging, numpy, kiwisolver, fonttools, cycler, matplotlib\n",
"Successfully installed cycler-0.11.0 fonttools-4.37.1 kiwisolver-1.4.4 matplotlib-3.5.3 numpy-1.21.6 packaging-21.3"
]
}
],
"source": [
"%%spark\n",
"\n",
"# To install a python package on the remote cluster (requests and matplotlib)\n",
"custom_pkg_install(['requests','matplotlib'], TYPE_PIP, INSTALL)"
]
},
{
"cell_type": "markdown",
"id": "82099b0b",
"metadata": {},
"source": [
"#### Import the above installed packages on the remote cluster"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "2259844f",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requests Version :: 2.28.1\n",
"MatPlotLib Version :: 3.5.3"
]
}
],
"source": [
"%%spark \n",
"\n",
"# Now import the above installed libs\n",
"import requests\n",
"import matplotlib\n",
"\n",
"print(f'Requests Version :: {requests.__version__}')\n",
"print(f'MatPlotLib Version :: {matplotlib.__version__}')"
]
},
{
"cell_type": "markdown",
"id": "603f8208",
"metadata": {},
"source": [
"#### Uninstall a package from PIP using the above method (on an attached cluster/runtime)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "c8339edf",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Running Command [sudo python3 -m pip uninstall requests matplotlib -y]\n",
"executing command on 10.0.1.220\n",
"Found existing installation: requests 2.28.1\n",
"Uninstalling requests-2.28.1:\n",
"Successfully uninstalled requests-2.28.1\n",
"Found existing installation: matplotlib 3.5.3\n",
"Uninstalling matplotlib-3.5.3:\n",
"Successfully uninstalled matplotlib-3.5.3\n",
"executing command on 10.0.1.201\n",
"Found existing installation: requests 2.28.1\n",
"Uninstalling requests-2.28.1:\n",
"Successfully uninstalled requests-2.28.1\n",
"Found existing installation: matplotlib 3.5.3\n",
"Uninstalling matplotlib-3.5.3:\n",
"Successfully uninstalled matplotlib-3.5.3\n",
"executing command on 10.0.1.119\n",
"Found existing installation: requests 2.28.1\n",
"Uninstalling requests-2.28.1:\n",
"Successfully uninstalled requests-2.28.1\n",
"Found existing installation: matplotlib 3.5.3\n",
"Uninstalling matplotlib-3.5.3:\n",
"Successfully uninstalled matplotlib-3.5.3"
]
}
],
"source": [
"%%spark\n",
"\n",
"# To uninstall a python package from the remote cluster (requests and matplotlib)\n",
"custom_pkg_install(['requests','matplotlib'], TYPE_PIP, UNINSTALL)"
]
},
{
"cell_type": "markdown",
"id": "6d80fb53",
"metadata": {},
"source": [
"#### Install a package from YUM using the above method (on an attached cluster/runtime)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "9317c41c",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Running Command [sudo yum install python3-devel python3-tools -y]\n",
"executing command on 10.0.1.220\n",
"Loaded plugins: extras_suggestions, langpacks, priorities, update-motd\n",
"10 packages excluded due to repository priority protections\n",
"Resolving Dependencies\n",
"--> Running transaction check\n",
"---> Package python3-devel.x86_64 0:3.7.10-1.amzn2.0.1 will be installed\n",
"--> Processing Dependency: python3-libs(x86-64) = 3.7.10-1.amzn2.0.1 for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Processing Dependency: python3 = 3.7.10-1.amzn2.0.1 for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Processing Dependency: python3-rpm-macros for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"---> Package python3-tools.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"--> Processing Dependency: python3-tools = 3.7.9-1.amzn2.0.1 for package: python3-test-3.7.9-1.amzn2.0.1.x86_64\n",
"---> Package python3-tools.x86_64 0:3.7.9-1.amzn2.0.1 will be obsoleted\n",
"---> Package python3-tools.x86_64 0:3.7.10-1.amzn2.0.1 will be obsoleting\n",
"--> Processing Dependency: python3-tkinter = 3.7.10-1.amzn2.0.1 for package: python3-tools-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Running transaction check\n",
"---> Package python3.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-libs.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-libs.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-rpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"--> Processing Dependency: python-srpm-macros >= 3-38 for package: python3-rpm-macros-3-60.amzn2.0.1.noarch\n",
"--> Processing Dependency: python-rpm-macros for package: python3-rpm-macros-3-60.amzn2.0.1.noarch\n",
"---> Package python3-test.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-test.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-tkinter.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-tkinter.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"--> Running transaction check\n",
"---> Package python-rpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"---> Package python-srpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"--> Finished Dependency Resolution\n",
"\n",
"Dependencies Resolved\n",
"\n",
"================================================================================\n",
"Package Arch Version Repository Size\n",
"================================================================================\n",
"Installing:\n",
"python3-devel x86_64 3.7.10-1.amzn2.0.1 amzn2-core 243 k\n",
"python3-tools x86_64 3.7.10-1.amzn2.0.1 amzn2-core 846 k\n",
"replacing python3-tools.x86_64 3.7.9-1.amzn2.0.1\n",
"Installing for dependencies:\n",
"python-rpm-macros noarch 3-60.amzn2.0.1 amzn2-core 14 k\n",
"python-srpm-macros noarch 3-60.amzn2.0.1 amzn2-core 18 k\n",
"python3-rpm-macros noarch 3-60.amzn2.0.1 amzn2-core 12 k\n",
"Updating for dependencies:\n",
"python3 x86_64 3.7.10-1.amzn2.0.1 amzn2-core 72 k\n",
"python3-libs x86_64 3.7.10-1.amzn2.0.1 amzn2-core 9.2 M\n",
"python3-test x86_64 3.7.10-1.amzn2.0.1 amzn2-core 7.6 M\n",
"python3-tkinter x86_64 3.7.10-1.amzn2.0.1 amzn2-core 372 k\n",
"\n",
"Transaction Summary\n",
"================================================================================\n",
"Install 2 Packages (+3 Dependent packages)\n",
"Upgrade ( 4 Dependent packages)\n",
"\n",
"Total download size: 18 M\n",
"Downloading packages:\n",
"Delta RPMs disabled because /usr/bin/applydeltarpm not installed.\n",
"--------------------------------------------------------------------------------\n",
"Total 58 MB/s | 18 MB 00:00\n",
"Running transaction check\n",
"Running transaction test\n",
"Transaction test succeeded\n",
"Running transaction\n",
"Updating : python3-3.7.10-1.amzn2.0.1.x86_64 1/14\n",
"Updating : python3-libs-3.7.10-1.amzn2.0.1.x86_64 2/14\n",
"Installing : python-srpm-macros-3-60.amzn2.0.1.noarch 3/14\n",
"Installing : python-rpm-macros-3-60.amzn2.0.1.noarch 4/14\n",
"Installing : python3-rpm-macros-3-60.amzn2.0.1.noarch 5/14\n",
"Updating : python3-tkinter-3.7.10-1.amzn2.0.1.x86_64 6/14\n",
"Installing : python3-tools-3.7.10-1.amzn2.0.1.x86_64 7/14\n",
"Updating : python3-test-3.7.10-1.amzn2.0.1.x86_64 8/14\n",
"Installing : python3-devel-3.7.10-1.amzn2.0.1.x86_64 9/14\n",
"Cleanup : python3-test-3.7.9-1.amzn2.0.1.x86_64 10/14\n",
"Cleanup : python3-tools-3.7.9-1.amzn2.0.1.x86_64 11/14\n",
"Cleanup : python3-tkinter-3.7.9-1.amzn2.0.1.x86_64 12/14\n",
"Cleanup : python3-3.7.9-1.amzn2.0.1.x86_64 13/14\n",
"Cleanup : python3-libs-3.7.9-1.amzn2.0.1.x86_64 14/14\n",
"Verifying : python3-libs-3.7.10-1.amzn2.0.1.x86_64 1/14\n",
"Verifying : python3-devel-3.7.10-1.amzn2.0.1.x86_64 2/14\n",
"Verifying : python3-tkinter-3.7.10-1.amzn2.0.1.x86_64 3/14\n",
"Verifying : python-rpm-macros-3-60.amzn2.0.1.noarch 4/14\n",
"Verifying : python-srpm-macros-3-60.amzn2.0.1.noarch 5/14\n",
"Verifying : python3-rpm-macros-3-60.amzn2.0.1.noarch 6/14\n",
"Verifying : python3-3.7.10-1.amzn2.0.1.x86_64 7/14\n",
"Verifying : python3-tools-3.7.10-1.amzn2.0.1.x86_64 8/14\n",
"Verifying : python3-test-3.7.10-1.amzn2.0.1.x86_64 9/14\n",
"Verifying : python3-tools-3.7.9-1.amzn2.0.1.x86_64 10/14\n",
"Verifying : python3-tools-3.7.9-1.amzn2.0.1.x86_64 11/14\n",
"Verifying : python3-3.7.9-1.amzn2.0.1.x86_64 12/14\n",
"Verifying : python3-tkinter-3.7.9-1.amzn2.0.1.x86_64 13/14\n",
"Verifying : python3-test-3.7.9-1.amzn2.0.1.x86_64 14/14\n",
"Verifying : python3-libs-3.7.9-1.amzn2.0.1.x86_64 15/14\n",
"\n",
"Installed:\n",
"python3-devel.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-tools.x86_64 0:3.7.10-1.amzn2.0.1\n",
"\n",
"Dependency Installed:\n",
"python-rpm-macros.noarch 0:3-60.amzn2.0.1\n",
"python-srpm-macros.noarch 0:3-60.amzn2.0.1\n",
"python3-rpm-macros.noarch 0:3-60.amzn2.0.1\n",
"\n",
"Dependency Updated:\n",
"python3.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-libs.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-test.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-tkinter.x86_64 0:3.7.10-1.amzn2.0.1\n",
"\n",
"Replaced:\n",
"python3-tools.x86_64 0:3.7.9-1.amzn2.0.1\n",
"\n",
"Complete!\n",
"executing command on 10.0.1.201\n",
"Loaded plugins: extras_suggestions, langpacks, priorities, update-motd\n",
"10 packages excluded due to repository priority protections\n",
"Resolving Dependencies\n",
"--> Running transaction check\n",
"---> Package python3-devel.x86_64 0:3.7.10-1.amzn2.0.1 will be installed\n",
"--> Processing Dependency: python3-libs(x86-64) = 3.7.10-1.amzn2.0.1 for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Processing Dependency: python3 = 3.7.10-1.amzn2.0.1 for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Processing Dependency: python3-rpm-macros for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"---> Package python3-tools.x86_64 0:3.7.9-1.amzn2.0.1 will be obsoleted\n",
"--> Processing Dependency: python3-tools = 3.7.9-1.amzn2.0.1 for package: python3-test-3.7.9-1.amzn2.0.1.x86_64\n",
"---> Package python3-tools.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-tools.x86_64 0:3.7.10-1.amzn2.0.1 will be obsoleting\n",
"--> Processing Dependency: python3-tkinter = 3.7.10-1.amzn2.0.1 for package: python3-tools-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Running transaction check\n",
"---> Package python3.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-libs.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-libs.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-rpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"--> Processing Dependency: python-srpm-macros >= 3-38 for package: python3-rpm-macros-3-60.amzn2.0.1.noarch\n",
"--> Processing Dependency: python-rpm-macros for package: python3-rpm-macros-3-60.amzn2.0.1.noarch\n",
"---> Package python3-test.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-test.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-tkinter.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-tkinter.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"--> Running transaction check\n",
"---> Package python-rpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"---> Package python-srpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"--> Finished Dependency Resolution\n",
"\n",
"Dependencies Resolved\n",
"\n",
"================================================================================\n",
"Package Arch Version Repository Size\n",
"================================================================================\n",
"Installing:\n",
"python3-devel x86_64 3.7.10-1.amzn2.0.1 amzn2-core 243 k\n",
"python3-tools x86_64 3.7.10-1.amzn2.0.1 amzn2-core 846 k\n",
"replacing python3-tools.x86_64 3.7.9-1.amzn2.0.1\n",
"Installing for dependencies:\n",
"python-rpm-macros noarch 3-60.amzn2.0.1 amzn2-core 14 k\n",
"python-srpm-macros noarch 3-60.amzn2.0.1 amzn2-core 18 k\n",
"python3-rpm-macros noarch 3-60.amzn2.0.1 amzn2-core 12 k\n",
"Updating for dependencies:\n",
"python3 x86_64 3.7.10-1.amzn2.0.1 amzn2-core 72 k\n",
"python3-libs x86_64 3.7.10-1.amzn2.0.1 amzn2-core 9.2 M\n",
"python3-test x86_64 3.7.10-1.amzn2.0.1 amzn2-core 7.6 M\n",
"python3-tkinter x86_64 3.7.10-1.amzn2.0.1 amzn2-core 372 k\n",
"\n",
"Transaction Summary\n",
"================================================================================\n",
"Install 2 Packages (+3 Dependent packages)\n",
"Upgrade ( 4 Dependent packages)\n",
"\n",
"Total download size: 18 M\n",
"Downloading packages:\n",
"Delta RPMs disabled because /usr/bin/applydeltarpm not installed.\n",
"--------------------------------------------------------------------------------\n",
"Total 63 MB/s | 18 MB 00:00\n",
"Running transaction check\n",
"Running transaction test\n",
"Transaction test succeeded\n",
"Running transaction\n",
"Updating : python3-3.7.10-1.amzn2.0.1.x86_64 1/14\n",
"Updating : python3-libs-3.7.10-1.amzn2.0.1.x86_64 2/14\n",
"Installing : python-srpm-macros-3-60.amzn2.0.1.noarch 3/14\n",
"Installing : python-rpm-macros-3-60.amzn2.0.1.noarch 4/14\n",
"Installing : python3-rpm-macros-3-60.amzn2.0.1.noarch 5/14\n",
"Updating : python3-tkinter-3.7.10-1.amzn2.0.1.x86_64 6/14\n",
"Installing : python3-tools-3.7.10-1.amzn2.0.1.x86_64 7/14\n",
"Updating : python3-test-3.7.10-1.amzn2.0.1.x86_64 8/14\n",
"Installing : python3-devel-3.7.10-1.amzn2.0.1.x86_64 9/14\n",
"Cleanup : python3-test-3.7.9-1.amzn2.0.1.x86_64 10/14\n",
"Cleanup : python3-tools-3.7.9-1.amzn2.0.1.x86_64 11/14\n",
"Cleanup : python3-tkinter-3.7.9-1.amzn2.0.1.x86_64 12/14\n",
"Cleanup : python3-3.7.9-1.amzn2.0.1.x86_64 13/14\n",
"Cleanup : python3-libs-3.7.9-1.amzn2.0.1.x86_64 14/14\n",
"Verifying : python3-libs-3.7.10-1.amzn2.0.1.x86_64 1/14\n",
"Verifying : python3-devel-3.7.10-1.amzn2.0.1.x86_64 2/14\n",
"Verifying : python3-tkinter-3.7.10-1.amzn2.0.1.x86_64 3/14\n",
"Verifying : python-rpm-macros-3-60.amzn2.0.1.noarch 4/14\n",
"Verifying : python-srpm-macros-3-60.amzn2.0.1.noarch 5/14\n",
"Verifying : python3-rpm-macros-3-60.amzn2.0.1.noarch 6/14\n",
"Verifying : python3-3.7.10-1.amzn2.0.1.x86_64 7/14\n",
"Verifying : python3-tools-3.7.10-1.amzn2.0.1.x86_64 8/14\n",
"Verifying : python3-test-3.7.10-1.amzn2.0.1.x86_64 9/14\n",
"Verifying : python3-tools-3.7.9-1.amzn2.0.1.x86_64 10/14\n",
"Verifying : python3-tools-3.7.9-1.amzn2.0.1.x86_64 11/14\n",
"Verifying : python3-3.7.9-1.amzn2.0.1.x86_64 12/14\n",
"Verifying : python3-tkinter-3.7.9-1.amzn2.0.1.x86_64 13/14\n",
"Verifying : python3-test-3.7.9-1.amzn2.0.1.x86_64 14/14\n",
"Verifying : python3-libs-3.7.9-1.amzn2.0.1.x86_64 15/14\n",
"\n",
"Installed:\n",
"python3-devel.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-tools.x86_64 0:3.7.10-1.amzn2.0.1\n",
"\n",
"Dependency Installed:\n",
"python-rpm-macros.noarch 0:3-60.amzn2.0.1\n",
"python-srpm-macros.noarch 0:3-60.amzn2.0.1\n",
"python3-rpm-macros.noarch 0:3-60.amzn2.0.1\n",
"\n",
"Dependency Updated:\n",
"python3.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-libs.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-test.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-tkinter.x86_64 0:3.7.10-1.amzn2.0.1\n",
"\n",
"Replaced:\n",
"python3-tools.x86_64 0:3.7.9-1.amzn2.0.1\n",
"\n",
"Complete!\n",
"executing command on 10.0.1.119\n",
"Loaded plugins: extras_suggestions, langpacks, priorities, update-motd\n",
"10 packages excluded due to repository priority protections\n",
"Resolving Dependencies\n",
"--> Running transaction check\n",
"---> Package python3-devel.x86_64 0:3.7.10-1.amzn2.0.1 will be installed\n",
"--> Processing Dependency: python3-libs(x86-64) = 3.7.10-1.amzn2.0.1 for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Processing Dependency: python3 = 3.7.10-1.amzn2.0.1 for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Processing Dependency: python3-rpm-macros for package: python3-devel-3.7.10-1.amzn2.0.1.x86_64\n",
"---> Package python3-tools.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"--> Processing Dependency: python3-tools = 3.7.9-1.amzn2.0.1 for package: python3-test-3.7.9-1.amzn2.0.1.x86_64\n",
"---> Package python3-tools.x86_64 0:3.7.9-1.amzn2.0.1 will be obsoleted\n",
"---> Package python3-tools.x86_64 0:3.7.10-1.amzn2.0.1 will be obsoleting\n",
"--> Processing Dependency: python3-tkinter = 3.7.10-1.amzn2.0.1 for package: python3-tools-3.7.10-1.amzn2.0.1.x86_64\n",
"--> Running transaction check\n",
"---> Package python3.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-libs.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-libs.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-rpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"--> Processing Dependency: python-srpm-macros >= 3-38 for package: python3-rpm-macros-3-60.amzn2.0.1.noarch\n",
"--> Processing Dependency: python-rpm-macros for package: python3-rpm-macros-3-60.amzn2.0.1.noarch\n",
"---> Package python3-test.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-test.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"---> Package python3-tkinter.x86_64 0:3.7.9-1.amzn2.0.1 will be updated\n",
"---> Package python3-tkinter.x86_64 0:3.7.10-1.amzn2.0.1 will be an update\n",
"--> Running transaction check\n",
"---> Package python-rpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"---> Package python-srpm-macros.noarch 0:3-60.amzn2.0.1 will be installed\n",
"--> Finished Dependency Resolution\n",
"\n",
"Dependencies Resolved\n",
"\n",
"================================================================================\n",
"Package Arch Version Repository Size\n",
"================================================================================\n",
"Installing:\n",
"python3-devel x86_64 3.7.10-1.amzn2.0.1 amzn2-core 243 k\n",
"python3-tools x86_64 3.7.10-1.amzn2.0.1 amzn2-core 846 k\n",
"replacing python3-tools.x86_64 3.7.9-1.amzn2.0.1\n",
"Installing for dependencies:\n",
"python-rpm-macros noarch 3-60.amzn2.0.1 amzn2-core 14 k\n",
"python-srpm-macros noarch 3-60.amzn2.0.1 amzn2-core 18 k\n",
"python3-rpm-macros noarch 3-60.amzn2.0.1 amzn2-core 12 k\n",
"Updating for dependencies:\n",
"python3 x86_64 3.7.10-1.amzn2.0.1 amzn2-core 72 k\n",
"python3-libs x86_64 3.7.10-1.amzn2.0.1 amzn2-core 9.2 M\n",
"python3-test x86_64 3.7.10-1.amzn2.0.1 amzn2-core 7.6 M\n",
"python3-tkinter x86_64 3.7.10-1.amzn2.0.1 amzn2-core 372 k\n",
"\n",
"Transaction Summary\n",
"================================================================================\n",
"Install 2 Packages (+3 Dependent packages)\n",
"Upgrade ( 4 Dependent packages)\n",
"\n",
"Total download size: 18 M\n",
"Downloading packages:\n",
"Delta RPMs disabled because /usr/bin/applydeltarpm not installed.\n",
"--------------------------------------------------------------------------------\n",
"Total 61 MB/s | 18 MB 00:00\n",
"Running transaction check\n",
"Running transaction test\n",
"Transaction test succeeded\n",
"Running transaction\n",
"Updating : python3-3.7.10-1.amzn2.0.1.x86_64 1/14\n",
"Updating : python3-libs-3.7.10-1.amzn2.0.1.x86_64 2/14\n",
"Installing : python-srpm-macros-3-60.amzn2.0.1.noarch 3/14\n",
"Installing : python-rpm-macros-3-60.amzn2.0.1.noarch 4/14\n",
"Installing : python3-rpm-macros-3-60.amzn2.0.1.noarch 5/14\n",
"Updating : python3-tkinter-3.7.10-1.amzn2.0.1.x86_64 6/14\n",
"Installing : python3-tools-3.7.10-1.amzn2.0.1.x86_64 7/14\n",
"Updating : python3-test-3.7.10-1.amzn2.0.1.x86_64 8/14\n",
"Installing : python3-devel-3.7.10-1.amzn2.0.1.x86_64 9/14\n",
"Cleanup : python3-test-3.7.9-1.amzn2.0.1.x86_64 10/14\n",
"Cleanup : python3-tools-3.7.9-1.amzn2.0.1.x86_64 11/14\n",
"Cleanup : python3-tkinter-3.7.9-1.amzn2.0.1.x86_64 12/14\n",
"Cleanup : python3-3.7.9-1.amzn2.0.1.x86_64 13/14\n",
"Cleanup : python3-libs-3.7.9-1.amzn2.0.1.x86_64 14/14\n",
"Verifying : python3-libs-3.7.10-1.amzn2.0.1.x86_64 1/14\n",
"Verifying : python3-devel-3.7.10-1.amzn2.0.1.x86_64 2/14\n",
"Verifying : python3-tkinter-3.7.10-1.amzn2.0.1.x86_64 3/14\n",
"Verifying : python-rpm-macros-3-60.amzn2.0.1.noarch 4/14\n",
"Verifying : python-srpm-macros-3-60.amzn2.0.1.noarch 5/14\n",
"Verifying : python3-rpm-macros-3-60.amzn2.0.1.noarch 6/14\n",
"Verifying : python3-3.7.10-1.amzn2.0.1.x86_64 7/14\n",
"Verifying : python3-tools-3.7.10-1.amzn2.0.1.x86_64 8/14\n",
"Verifying : python3-test-3.7.10-1.amzn2.0.1.x86_64 9/14\n",
"Verifying : python3-tools-3.7.9-1.amzn2.0.1.x86_64 10/14\n",
"Verifying : python3-tools-3.7.9-1.amzn2.0.1.x86_64 11/14\n",
"Verifying : python3-3.7.9-1.amzn2.0.1.x86_64 12/14\n",
"Verifying : python3-tkinter-3.7.9-1.amzn2.0.1.x86_64 13/14\n",
"Verifying : python3-test-3.7.9-1.amzn2.0.1.x86_64 14/14\n",
"Verifying : python3-libs-3.7.9-1.amzn2.0.1.x86_64 15/14\n",
"\n",
"Installed:\n",
"python3-devel.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-tools.x86_64 0:3.7.10-1.amzn2.0.1\n",
"\n",
"Dependency Installed:\n",
"python-rpm-macros.noarch 0:3-60.amzn2.0.1\n",
"python-srpm-macros.noarch 0:3-60.amzn2.0.1\n",
"python3-rpm-macros.noarch 0:3-60.amzn2.0.1\n",
"\n",
"Dependency Updated:\n",
"python3.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-libs.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-test.x86_64 0:3.7.10-1.amzn2.0.1\n",
"python3-tkinter.x86_64 0:3.7.10-1.amzn2.0.1\n",
"\n",
"Replaced:\n",
"python3-tools.x86_64 0:3.7.9-1.amzn2.0.1\n",
"\n",
"Complete!"
]
}
],
"source": [
"%%spark\n",
"\n",
"# To install a yum package (python3-devel and python3-tools)\n",
"custom_pkg_install(['python3-devel','python3-tools'], TYPE_YUM, INSTALL)"
]
},
{
"cell_type": "markdown",
"id": "f70df466",
"metadata": {},
"source": [
"#### Uninstall a package from YUM using the above method (on an attached cluster/runtime)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "d5ac22f0",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Running Command [sudo yum uninstall python3-devel python3-tools -y]\n",
"executing command on 10.0.1.220\n",
"Loaded plugins: extras_suggestions, langpacks, priorities, update-motd\n",
"executing command on 10.0.1.201\n",
"Loaded plugins: extras_suggestions, langpacks, priorities, update-motd\n",
"executing command on 10.0.1.119\n",
"Loaded plugins: extras_suggestions, langpacks, priorities, update-motd"
]
}
],
"source": [
"%%spark\n",
"\n",
"# To uninstall a yum package (python3-devel and python3-tools)\n",
"custom_pkg_install(['python3-devel','python3-tools'], TYPE_YUM, UNINSTALL)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Syntasa Kernel",
"language": "python",
"name": "syntasa_kernel"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
},
"syn_metadata": {
"spark_lang_type": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}