MatrixMultiplication_SystemML.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lw448Q1BJ7Gm",
        "outputId": "ec774609-88b5-47ec-a428-88955e8fc7a0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: systemml in /usr/local/lib/python3.10/dist-packages (1.2.0)\n",
            "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.10/dist-packages (from systemml) (1.25.2)\n",
            "Requirement already satisfied: scipy>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from systemml) (1.11.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from systemml) (2.0.3)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from systemml) (1.2.2)\n",
            "Requirement already satisfied: Pillow>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from systemml) (9.4.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->systemml) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->systemml) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->systemml) (2024.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->systemml) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->systemml) (3.5.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->systemml) (1.16.0)\n",
            "Requirement already satisfied: pyspark in /usr/local/lib/python3.10/dist-packages (3.5.1)\n",
            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n",
            "Requirement already satisfied: findspark in /usr/local/lib/python3.10/dist-packages (2.0.1)\n"
          ]
        }
      ],
      "source": [
        "!pip install systemml\n",
        "!pip install pyspark\n",
        "!pip install findspark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import findspark\n",
        "findspark.init()\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()\n"
      ],
      "metadata": {
        "id": "jeg99MMtTR3C"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from systemml import MLContext, dml\n",
        "ml = MLContext(spark)\n",
        "print(ml.info())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5S574bPxKo_2",
        "outputId": "b6147df0-b4fc-40ba-d068-f44143b3a312"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archiver-Version: Plexus Archiver\n",
            "Artifact-Id: systemml\n",
            "Build-Jdk: 1.8.0_151\n",
            "Build-Time: 2018-08-17 05:58:31 UTC\n",
            "Built-By: reinwald\n",
            "Created-By: Apache Maven 3.3.9\n",
            "Group-Id: org.apache.systemml\n",
            "Main-Class: org.apache.sysml.api.DMLScript\n",
            "Manifest-Version: 1.0\n",
            "Minimum-Recommended-Spark-Version: 2.1.0\n",
            "Version: 1.2.0\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "script = dml(\"\"\"\n",
        "print('Hello World')\"\"\")\n",
        "ml.execute(script)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GPTn6HDHX54q",
        "outputId": "a43426b0-f2ab-468e-9143-441a7da3c957"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ANTLR Tool version 4.5.3 used for code generation does not match the current runtime version 4.8ANTLR Runtime version 4.5.3 used for parser compilation does not match the current runtime version 4.8ANTLR Tool version 4.5.3 used for code generation does not match the current runtime version 4.8ANTLR Runtime version 4.5.3 used for parser compilation does not match the current runtime version 4.8\n",
            "Hello World\n",
            "SystemML Statistics:\n",
            "Total execution time:\t\t0.003 sec.\n",
            "Number of executed Spark inst:\t0.\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MLResults"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "script = dml(\"\"\"\n",
        "s = 'Hello World' \"\"\").output(\"s\")\n",
        "hello_world_str = ml.execute(script).get(\"s\")\n",
        "print(hello_world_str)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GtH0R2tmYRpQ",
        "outputId": "7d5d2462-1e9a-41b1-e3fc-228932e58a93"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "SystemML Statistics:\n",
            "Total execution time:\t\t0.000 sec.\n",
            "Number of executed Spark inst:\t0.\n",
            "\n",
            "\n",
            "Hello World\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import numpy, sklearn, and some helper functions"
      ],
      "metadata": {
        "id": "ITKFPmuMY1ys"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sys, os\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from sklearn import datasets\n",
        "plt.switch_backend('agg')"
      ],
      "metadata": {
        "id": "BcJO6wQdY-TB"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Matrix multiplication"
      ],
      "metadata": {
        "id": "fd7f7MvRZM6Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "script = \"\"\"  X = rand(rows = $nr, cols =1000, sparsity = 0.5)\n",
        "A = t(X) %*% X\n",
        "s = sum(A)\n",
        "\"\"\"\n",
        "prog = dml(script).input('$nr',1e6).output('s')\n",
        "s = ml.execute(prog).get('s')\n",
        "print(s)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NOizUS3eZOxR",
        "outputId": "6e958725-e2d5-413c-fafc-70fc4f4ccb16"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Stage 0:>                                                                             (0 + 2) / 59]\n",
            "[Stage 0:==>                                                                           (2 + 2) / 59]\n",
            "[Stage 0:=====>                                                                        (4 + 2) / 59]\n",
            "[Stage 0:=======>                                                                      (6 + 2) / 59]\n",
            "[Stage 0:==========>                                                                   (8 + 2) / 59]\n",
            "[Stage 0:=============>                                                               (10 + 2) / 59]\n",
            "[Stage 0:===============>                                                             (12 + 2) / 59]\n",
            "[Stage 0:==================>                                                          (14 + 2) / 59]\n",
            "[Stage 0:====================>                                                        (16 + 2) / 59]\n",
            "[Stage 0:======================>                                                      (17 + 2) / 59]\n",
            "[Stage 0:=======================>                                                     (18 + 2) / 59]\n",
            "[Stage 0:==========================>                                                  (20 + 2) / 59]\n",
            "[Stage 0:============================>                                                (22 + 2) / 59]\n",
            "[Stage 0:==============================>                                              (23 + 2) / 59]\n",
            "[Stage 0:===============================>                                             (24 + 2) / 59]\n",
            "[Stage 0:=================================>                                           (26 + 2) / 59]\n",
            "[Stage 0:===================================>                                         (27 + 2) / 59]\n",
            "[Stage 0:=====================================>                                       (29 + 2) / 59]\n",
            "[Stage 0:=======================================>                                     (30 + 2) / 59]\n",
            "[Stage 0:=========================================>                                   (32 + 2) / 59]\n",
            "[Stage 0:============================================>                                (34 + 2) / 59]\n",
            "[Stage 0:=============================================>                               (35 + 2) / 59]\n",
            "[Stage 0:==============================================>                              (36 + 2) / 59]\n",
            "[Stage 0:=================================================>                           (38 + 2) / 59]\n",
            "[Stage 0:====================================================>                        (40 + 2) / 59]\n",
            "[Stage 0:=====================================================>                       (41 + 2) / 59]\n",
            "[Stage 0:======================================================>                      (42 + 2) / 59]\n",
            "[Stage 0:========================================================>                    (43 + 2) / 59]\n",
            "[Stage 0:=========================================================>                   (44 + 2) / 59]\n",
            "[Stage 0:============================================================>                (46 + 2) / 59]\n",
            "[Stage 0:=============================================================>               (47 + 2) / 59]\n",
            "[Stage 0:==============================================================>              (48 + 2) / 59]\n",
            "[Stage 0:=================================================================>           (50 + 2) / 59]\n",
            "[Stage 0:===================================================================>         (52 + 2) / 59]\n",
            "[Stage 0:======================================================================>      (54 + 2) / 59]\n",
            "[Stage 0:=========================================================================>   (56 + 2) / 59]\n",
            "[Stage 0:===========================================================================> (58 + 1) / 59]\n",
            "SystemML Statistics:\n",
            "Total execution time:\t\t45.564 sec.\n",
            "Number of executed Spark inst:\t2.\n",
            "\n",
            "                                                                                                    \n",
            "\n",
            "62606171865.332375\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vk1_WU46ajTM"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}