Merge pull request #5 from Azure/Task_Restructure_Pip

Code restructured and pip installation

analytics_notebook/adlmagics/adlmagics/00_adlmagics_demo.ipynb → adlmagics/00_adlmagics_demo.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -825,7 +825,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "version": "3.6.5"
   }
  },
  "nbformat": 4,

adlmagics/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

adlmagics/README.md

@@ -0,0 +1,53 @@
+
+# Azure Data Service Notebook
+Azure Data Service Notebook is a set of tool for working with Azure Data Service (including HDInsight, Azure Data Lake, etc)
+
+
+# Feature
+Azure Data Service Notebook currently provides a set of magic commands for users to access Azure Data Lake. Available magics are captured in the table below.
+
+| Commands | Function |
+|-----|-----|
+|%adl login |Line magic to log in to Azure Data Lake.|
+|%adl listaccounts|Line magic to list the Azure Data Lake analytic accounts for current user.|
+|%adl listjobs|Line magic to list the Azure Data Lake jobs for a given account.|
+|%%adl submitjob|Cell magic to submit a USQL job to ADL cluster.|
+|%adl viewjob|Line magic to view job info.|
+|%adl liststorefile|Line magic to list the Azure Data Lake store accounts.|
+|%adl sample|Line magic to sample a given file, return results as Pandas DataFrame.|
+|%adl logout|Line magic to log out.|
+
+
+# Installation
+
+- Download and Install [python 3.6+](https://www.python.org/downloads/)
+- Install jupyter: `pip install jupyter` 
+- Install azure packages fully : 
+	`pip install azure`
+-  Or just install the key packages : 
+	- `pip install adla`
+	- `pip install azure-mgmt-resource`
+	- `pip install azure-mgmt-datalake-analytics` 
+	- `pip install azure-mgmt-datalake-store` 
+	- `pip install azure-datalake-store` 
+- Copy the [adlmagic folder](/analytics_notebook/adlmagics) to local box. 
+
+
+# Examples
+- [adlmagics_demo.ipynb](/analytics_notebook/adlmagics/00_adlmagics_demo.ipynb), demo file of Azure Data Lake job control functions.
+
+
+
+# Contributing
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.microsoft.com.
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.

adlmagics/adlmagics/__init__.py

@@ -0,0 +1,4 @@
+from adlmagics.adlmagics_main import AdlMagics
+
+def load_ipython_extension(ipython):
+    ipython.register_magics(AdlMagics)

analytics_notebook/adlmagics/adlmagics/adlmagics.py → adlmagics/adlmagics/adlmagics_main.py

@@ -2,25 +2,25 @@
 from sys import stdout
 from os import linesep
 
-from services.azure_token_service import AzureTokenService
-from services.adla_service_sdk_impl import AdlaServiceSdkImpl
-from services.adls_service_sdk_impl import AdlsServiceSdkImpl
-
-from magics.azure.azure_magic_base import AzureMagicBase
-from magics.azure.azure_login_magic import AzureLoginMagic
-from magics.azure.azure_logout_magic import AzureLogoutMagic
-
-from magics.adla.adla_magic_base import AdlaMagicBase
-from magics.adla.adla_accounts_listing_magic import AdlaAccountsListingMagic
-from magics.adla.adla_job_viewing_magic import AdlaJobViewingMagic
-from magics.adla.adla_job_submission_magic import AdlaJobSubmissionMagic
-from magics.adla.adla_jobs_listing_magic import AdlaJobsListingMagic
-
-from magics.adls.adls_magic_base import AdlsMagicBase
-from magics.adls.adls_accounts_listing_magic import AdlsAccountsListingMagic
-from magics.adls.adls_folders_listing_magic import AdlsFoldersListingMagic
-from magics.adls.adls_files_listing_magic import AdlsFilesListingMagic
-from magics.adls.adls_file_sampling_magic import AdlsFileSamplingMagic
+from adlmagics.services.azure_token_service import AzureTokenService
+from adlmagics.services.adla_service_sdk_impl import AdlaServiceSdkImpl
+from adlmagics.services.adls_service_sdk_impl import AdlsServiceSdkImpl
+
+from adlmagics.magics.azure.azure_magic_base import AzureMagicBase
+from adlmagics.magics.azure.azure_login_magic import AzureLoginMagic
+from adlmagics.magics.azure.azure_logout_magic import AzureLogoutMagic
+
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.magics.adla.adla_accounts_listing_magic import AdlaAccountsListingMagic
+from adlmagics.magics.adla.adla_job_viewing_magic import AdlaJobViewingMagic
+from adlmagics.magics.adla.adla_job_submission_magic import AdlaJobSubmissionMagic
+from adlmagics.magics.adla.adla_jobs_listing_magic import AdlaJobsListingMagic
+
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_accounts_listing_magic import AdlsAccountsListingMagic
+from adlmagics.magics.adls.adls_folders_listing_magic import AdlsFoldersListingMagic
+from adlmagics.magics.adls.adls_files_listing_magic import AdlsFilesListingMagic
+from adlmagics.magics.adls.adls_file_sampling_magic import AdlsFileSamplingMagic
 
 @magics_class
 class AdlMagics(Magics):
@@ -88,7 +88,4 @@ def __register_adls_magic(self, adls_magic_class):
         self.__magics[adls_magic.cmd_name.lower()] = adls_magic
 
     def __write_line(self, text):
-        stdout.write(text + linesep)
-
-def load_ipython_extension(ipython):
-    ipython.register_magics(AdlMagics)
+        stdout.write(text + linesep)

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py → adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
 
 class AdlaAccountsListingMagic(AdlaMagicBase):
     def __init__(self, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py → adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py

@@ -1,7 +1,7 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adla.adla_magic_base import AdlaMagicBase
-from models.adla_job_submission import AdlaJobSubmission
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.models.adla_job_submission import AdlaJobSubmission
 
 class AdlaJobSubmissionMagic(AdlaMagicBase):
     def __init__(self, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py → adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
 
 class AdlaJobViewingMagic(AdlaMagicBase):
     def __init__(self, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py → adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py

@@ -1,7 +1,7 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 from os import linesep
 
-from magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
 
 class AdlaJobsListingMagic(AdlaMagicBase):
     def __init__(self, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_magic_base.py → adlmagics/adlmagics/magics/adla/adla_magic_base.py

@@ -1,4 +1,4 @@
-from magics.magic_base import MagicBase
+from adlmagics.magics.magic_base import MagicBase
 
 class AdlaMagicBase(MagicBase):
     def __init__(self, cmd_name, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py → adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
 
 class AdlsAccountsListingMagic(AdlsMagicBase):
     def __init__(self, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py → adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py

@@ -1,7 +1,7 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 from pandas import DataFrame
 
-from magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
 
 class AdlsFileSamplingMagic(AdlsMagicBase):
     def __init__(self, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py → adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
 
 class AdlsFilesListingMagic(AdlsMagicBase):
     def __init__(self, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py → adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
 
 class AdlsFoldersListingMagic(AdlsMagicBase):
     def __init__(self, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_magic_base.py → adlmagics/adlmagics/magics/adls/adls_magic_base.py

@@ -1,4 +1,4 @@
-from magics.magic_base import MagicBase
+from adlmagics.magics.magic_base import MagicBase
 
 class AdlsMagicBase(MagicBase):
     def __init__(self, cmd_name, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/azure/azure_login_magic.py → adlmagics/adlmagics/magics/azure/azure_login_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.azure.azure_magic_base import AzureMagicBase
+from adlmagics.magics.azure.azure_magic_base import AzureMagicBase
 
 class AzureLoginMagic(AzureMagicBase):
     def __init__(self, token_service):

analytics_notebook/adlmagics/adlmagics/magics/azure/azure_logout_magic.py → adlmagics/adlmagics/magics/azure/azure_logout_magic.py

@@ -1,4 +1,4 @@
-from magics.azure.azure_magic_base import AzureMagicBase
+from adlmagics.magics.azure.azure_magic_base import AzureMagicBase
 
 class AzureLogoutMagic(AzureMagicBase):
     def __init__(self, token_service):

analytics_notebook/adlmagics/adlmagics/magics/azure/azure_magic_base.py → adlmagics/adlmagics/magics/azure/azure_magic_base.py

@@ -1,4 +1,4 @@
-from magics.magic_base import MagicBase
+from adlmagics.magics.magic_base import MagicBase
 
 class AzureMagicBase(MagicBase):
     def __init__(self, cmd_name, token_service):

analytics_notebook/adlmagics/adlmagics/services/adla_service_sdk_impl.py → adlmagics/adlmagics/services/adla_service_sdk_impl.py

@@ -4,10 +4,10 @@
 from azure.mgmt.datalake.analytics.job.models import JobInformation, USqlJobProperties, JobType
 from uuid import uuid4
 
-from exceptions import UserNotLoggedInError
-from magics.adla.adla_consts import AdlaJobConsts
-from models.adla_account import AdlaAccount
-from models.adla_job import AdlaJob
+from adlmagics.exceptions import UserNotLoggedInError
+from adlmagics.magics.adla.adla_consts import AdlaJobConsts
+from adlmagics.models.adla_account import AdlaAccount
+from adlmagics.models.adla_job import AdlaJob
 
 class AdlaServiceSdkImpl:
     def __init__(self, token_service):

analytics_notebook/adlmagics/adlmagics/services/adls_service_sdk_impl.py → adlmagics/adlmagics/services/adls_service_sdk_impl.py

@@ -7,10 +7,10 @@
 from datetime import datetime
 from sys import stdout
 
-from exceptions import UserNotLoggedInError
-from models.adls_account import AdlsAccount
-from models.adls_folder import AdlsFolder
-from models.adls_file import AdlsFile
+from adlmagics.exceptions import UserNotLoggedInError
+from adlmagics.models.adls_account import AdlsAccount
+from adlmagics.models.adls_folder import AdlsFolder
+from adlmagics.models.adls_file import AdlsFile
 
 class AdlsServiceSdkImpl:
     def __init__(self, token_service):
@@ -62,7 +62,7 @@ def read_file(self, account, file_path, encoding, column_sep, to_be_read_line_co
             for read_line in f:
                 read_lines.append(read_line.decode(encoding).strip().split(column_sep))
                 read_line_count += 1
-                if (read_line_count > to_be_read_line_count):
+                if (read_line_count >= to_be_read_line_count):
                     break
 
         return read_lines

adlmagics/requirements.txt

@@ -0,0 +1,10 @@
+ipython>=5.3.0
+notebook>=5.0.0
+ipykernel>=4.6.1
+pandas>=0.20.1
+numpy
+adal
+azure-mgmt-resource
+azure-mgmt-datalake-analytics
+azure-mgmt-datalake-store
+azure-datalake-store

adlmagics/setup.py

@@ -0,0 +1,54 @@
+import setuptools
+
+name             = "adlmagics"
+description      = "Azure Data Lake management magics for Jupyter Notebook"
+long_description = ""
+version          = "0.0.1"
+packages         = ["adlmagics",
+                    "adlmagics/magics/adla",
+                    "adlmagics/magics/adls",
+                    "adlmagics/magics/azure",
+                    "adlmagics/magics/session",
+                    "adlmagics/models",
+                    "adlmagics/services"]
+author           = "Vincent Feng"
+author_email     = "ivincentfeng@live.com"
+url              = "https://github.com/Azure/Azure-Data-Service-Notebook"
+download_url     = "https://github.com/Azure/Azure-Data-Service-Notebook"
+license          = "MIT"
+
+with open("README.md", "r") as f:
+    long_description = f.read()
+
+setuptools.setup(
+  name = name,
+  version = version,
+  description = description,
+  long_description = long_description,
+  long_description_content_type = "text/markdown",
+  author = author,
+  author_email = author_email,
+  url = url,
+  download_url = download_url,
+  license = license,
+  packages = setuptools.find_packages(),
+  include_package_data = True,
+  classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Framework :: Jupyter",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Programming Language :: Python :: 3.6",
+    "Operating System :: OS Independent"],
+  install_requires = [
+    "ipython>=5.3.0",
+    "notebook>=5.0.0",
+    "ipykernel>=4.6.1",
+    "pandas>=0.20.1",
+    "numpy",
+    "adal",
+    "azure-mgmt-resource",
+    "azure-mgmt-datalake-analytics",
+    "azure-mgmt-datalake-store",
+    "azure-datalake-store"])