Merge pull request #5 from Azure/Task_Restructure_Pip

Code restructured and pip installation

analytics_notebook/adlmagics/adlmagics/00_adlmagics_demo.ipynb → adlmagics/00_adlmagics_demo.ipynb

@@ -2,7 +2,7 @@
  "cells": [
  {
  "cell_type": "code",
- "execution_count": 3,
+ "execution_count": 1,
  "metadata": {},
  "outputs": [
  {
@@ -825,7 +825,7 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.6.3"
+ "version": "3.6.5"
  }
  },
  "nbformat": 4,

adlmagics/LICENSE

@@ -0,0 +1,21 @@
+ MIT License
+
+ Copyright (c) Microsoft Corporation. All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE

adlmagics/README.md

@@ -0,0 +1,53 @@
+
+# Azure Data Service Notebook
+Azure Data Service Notebook is a set of tool for working with Azure Data Service (including HDInsight, Azure Data Lake, etc)
+
+
+# Feature
+Azure Data Service Notebook currently provides a set of magic commands for users to access Azure Data Lake. Available magics are captured in the table below.
+
+| Commands | Function |
+|-----|-----|
+|%adl login |Line magic to log in to Azure Data Lake.|
+|%adl listaccounts|Line magic to list the Azure Data Lake analytic accounts for current user.|
+|%adl listjobs|Line magic to list the Azure Data Lake jobs for a given account.|
+|%%adl submitjob|Cell magic to submit a USQL job to ADL cluster.|
+|%adl viewjob|Line magic to view job info.|
+|%adl liststorefile|Line magic to list the Azure Data Lake store accounts.|
+|%adl sample|Line magic to sample a given file, return results as Pandas DataFrame.|
+|%adl logout|Line magic to log out.|
+
+
+# Installation
+
+- Download and Install [python 3.6+](https://www.python.org/downloads/)
+- Install jupyter: `pip install jupyter` 
+- Install azure packages fully : 
+ `pip install azure`
+- Or just install the key packages : 
+ - `pip install adla`
+ - `pip install azure-mgmt-resource`
+ - `pip install azure-mgmt-datalake-analytics` 
+ - `pip install azure-mgmt-datalake-store` 
+ - `pip install azure-datalake-store` 
+- Copy the [adlmagic folder](/analytics_notebook/adlmagics) to local box. 
+
+
+# Examples
+- [adlmagics_demo.ipynb](/analytics_notebook/adlmagics/00_adlmagics_demo.ipynb), demo file of Azure Data Lake job control functions.
+
+
+
+# Contributing
+
+This project welcomes contributions and suggestions. Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.microsoft.com.
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.

adlmagics/adlmagics/__init__.py

@@ -0,0 +1,4 @@
+from adlmagics.adlmagics_main import AdlMagics
+
+def load_ipython_extension(ipython):
+ ipython.register_magics(AdlMagics)

analytics_notebook/adlmagics/adlmagics/adlmagics.py → adlmagics/adlmagics/adlmagics_main.py

@@ -2,25 +2,25 @@
 from sys import stdout
 from os import linesep
 
-from services.azure_token_service import AzureTokenService
-from services.adla_service_sdk_impl import AdlaServiceSdkImpl
-from services.adls_service_sdk_impl import AdlsServiceSdkImpl
-
-from magics.azure.azure_magic_base import AzureMagicBase
-from magics.azure.azure_login_magic import AzureLoginMagic
-from magics.azure.azure_logout_magic import AzureLogoutMagic
-
-from magics.adla.adla_magic_base import AdlaMagicBase
-from magics.adla.adla_accounts_listing_magic import AdlaAccountsListingMagic
-from magics.adla.adla_job_viewing_magic import AdlaJobViewingMagic
-from magics.adla.adla_job_submission_magic import AdlaJobSubmissionMagic
-from magics.adla.adla_jobs_listing_magic import AdlaJobsListingMagic
-
-from magics.adls.adls_magic_base import AdlsMagicBase
-from magics.adls.adls_accounts_listing_magic import AdlsAccountsListingMagic
-from magics.adls.adls_folders_listing_magic import AdlsFoldersListingMagic
-from magics.adls.adls_files_listing_magic import AdlsFilesListingMagic
-from magics.adls.adls_file_sampling_magic import AdlsFileSamplingMagic
+from adlmagics.services.azure_token_service import AzureTokenService
+from adlmagics.services.adla_service_sdk_impl import AdlaServiceSdkImpl
+from adlmagics.services.adls_service_sdk_impl import AdlsServiceSdkImpl
+
+from adlmagics.magics.azure.azure_magic_base import AzureMagicBase
+from adlmagics.magics.azure.azure_login_magic import AzureLoginMagic
+from adlmagics.magics.azure.azure_logout_magic import AzureLogoutMagic
+
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.magics.adla.adla_accounts_listing_magic import AdlaAccountsListingMagic
+from adlmagics.magics.adla.adla_job_viewing_magic import AdlaJobViewingMagic
+from adlmagics.magics.adla.adla_job_submission_magic import AdlaJobSubmissionMagic
+from adlmagics.magics.adla.adla_jobs_listing_magic import AdlaJobsListingMagic
+
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_accounts_listing_magic import AdlsAccountsListingMagic
+from adlmagics.magics.adls.adls_folders_listing_magic import AdlsFoldersListingMagic
+from adlmagics.magics.adls.adls_files_listing_magic import AdlsFilesListingMagic
+from adlmagics.magics.adls.adls_file_sampling_magic import AdlsFileSamplingMagic
 
 @magics_class
 class AdlMagics(Magics):
@@ -88,7 +88,4 @@ def __register_adls_magic(self, adls_magic_class):
  self.__magics[adls_magic.cmd_name.lower()] = adls_magic
 
  def __write_line(self, text):
- stdout.write(text + linesep)
-
-def load_ipython_extension(ipython):
- ipython.register_magics(AdlMagics)
+ stdout.write(text + linesep)

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py → adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
 
 class AdlaAccountsListingMagic(AdlaMagicBase):
  def __init__(self, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py → adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py

@@ -1,7 +1,7 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adla.adla_magic_base import AdlaMagicBase
-from models.adla_job_submission import AdlaJobSubmission
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.models.adla_job_submission import AdlaJobSubmission
 
 class AdlaJobSubmissionMagic(AdlaMagicBase):
  def __init__(self, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py → adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
 
 class AdlaJobViewingMagic(AdlaMagicBase):
  def __init__(self, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py → adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py

@@ -1,7 +1,7 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 from os import linesep
 
-from magics.adla.adla_magic_base import AdlaMagicBase
+from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase
 
 class AdlaJobsListingMagic(AdlaMagicBase):
  def __init__(self, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adla/adla_magic_base.py → adlmagics/adlmagics/magics/adla/adla_magic_base.py

@@ -1,4 +1,4 @@
-from magics.magic_base import MagicBase
+from adlmagics.magics.magic_base import MagicBase
 
 class AdlaMagicBase(MagicBase):
  def __init__(self, cmd_name, adla_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py → adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
 
 class AdlsAccountsListingMagic(AdlsMagicBase):
  def __init__(self, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py → adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py

@@ -1,7 +1,7 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 from pandas import DataFrame
 
-from magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
 
 class AdlsFileSamplingMagic(AdlsMagicBase):
  def __init__(self, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py → adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
 
 class AdlsFilesListingMagic(AdlsMagicBase):
  def __init__(self, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py → adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.adls.adls_magic_base import AdlsMagicBase
+from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase
 
 class AdlsFoldersListingMagic(AdlsMagicBase):
  def __init__(self, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/adls/adls_magic_base.py → adlmagics/adlmagics/magics/adls/adls_magic_base.py

@@ -1,4 +1,4 @@
-from magics.magic_base import MagicBase
+from adlmagics.magics.magic_base import MagicBase
 
 class AdlsMagicBase(MagicBase):
  def __init__(self, cmd_name, adls_service):

analytics_notebook/adlmagics/adlmagics/magics/azure/azure_login_magic.py → adlmagics/adlmagics/magics/azure/azure_login_magic.py

@@ -1,6 +1,6 @@
 from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring
 
-from magics.azure.azure_magic_base import AzureMagicBase
+from adlmagics.magics.azure.azure_magic_base import AzureMagicBase
 
 class AzureLoginMagic(AzureMagicBase):
  def __init__(self, token_service):

analytics_notebook/adlmagics/adlmagics/magics/azure/azure_logout_magic.py → adlmagics/adlmagics/magics/azure/azure_logout_magic.py

@@ -1,4 +1,4 @@
-from magics.azure.azure_magic_base import AzureMagicBase
+from adlmagics.magics.azure.azure_magic_base import AzureMagicBase
 
 class AzureLogoutMagic(AzureMagicBase):
  def __init__(self, token_service):

analytics_notebook/adlmagics/adlmagics/magics/azure/azure_magic_base.py → adlmagics/adlmagics/magics/azure/azure_magic_base.py

@@ -1,4 +1,4 @@
-from magics.magic_base import MagicBase
+from adlmagics.magics.magic_base import MagicBase
 
 class AzureMagicBase(MagicBase):
  def __init__(self, cmd_name, token_service):

analytics_notebook/adlmagics/adlmagics/services/adla_service_sdk_impl.py → adlmagics/adlmagics/services/adla_service_sdk_impl.py

@@ -4,10 +4,10 @@
 from azure.mgmt.datalake.analytics.job.models import JobInformation, USqlJobProperties, JobType
 from uuid import uuid4
 
-from exceptions import UserNotLoggedInError
-from magics.adla.adla_consts import AdlaJobConsts
-from models.adla_account import AdlaAccount
-from models.adla_job import AdlaJob
+from adlmagics.exceptions import UserNotLoggedInError
+from adlmagics.magics.adla.adla_consts import AdlaJobConsts
+from adlmagics.models.adla_account import AdlaAccount
+from adlmagics.models.adla_job import AdlaJob
 
 class AdlaServiceSdkImpl:
  def __init__(self, token_service):

analytics_notebook/adlmagics/adlmagics/services/adls_service_sdk_impl.py → adlmagics/adlmagics/services/adls_service_sdk_impl.py

@@ -7,10 +7,10 @@
 from datetime import datetime
 from sys import stdout
 
-from exceptions import UserNotLoggedInError
-from models.adls_account import AdlsAccount
-from models.adls_folder import AdlsFolder
-from models.adls_file import AdlsFile
+from adlmagics.exceptions import UserNotLoggedInError
+from adlmagics.models.adls_account import AdlsAccount
+from adlmagics.models.adls_folder import AdlsFolder
+from adlmagics.models.adls_file import AdlsFile
 
 class AdlsServiceSdkImpl:
  def __init__(self, token_service):
@@ -62,7 +62,7 @@ def read_file(self, account, file_path, encoding, column_sep, to_be_read_line_co
  for read_line in f:
  read_lines.append(read_line.decode(encoding).strip().split(column_sep))
  read_line_count += 1
- if (read_line_count > to_be_read_line_count):
+ if (read_line_count >= to_be_read_line_count):
  break
 
  return read_lines

adlmagics/requirements.txt

@@ -0,0 +1,10 @@
+ipython>=5.3.0
+notebook>=5.0.0
+ipykernel>=4.6.1
+pandas>=0.20.1
+numpy
+adal
+azure-mgmt-resource
+azure-mgmt-datalake-analytics
+azure-mgmt-datalake-store
+azure-datalake-store

adlmagics/setup.py

@@ -0,0 +1,54 @@
+import setuptools
+
+name = "adlmagics"
+description = "Azure Data Lake management magics for Jupyter Notebook"
+long_description = ""
+version = "0.0.1"
+packages = ["adlmagics",
+ "adlmagics/magics/adla",
+ "adlmagics/magics/adls",
+ "adlmagics/magics/azure",
+ "adlmagics/magics/session",
+ "adlmagics/models",
+ "adlmagics/services"]
+author = "Vincent Feng"
+author_email = "ivincentfeng@live.com"
+url = "https://github.com/Azure/Azure-Data-Service-Notebook"
+download_url = "https://github.com/Azure/Azure-Data-Service-Notebook"
+license = "MIT"
+
+with open("README.md", "r") as f:
+ long_description = f.read()
+
+setuptools.setup(
+ name = name,
+ version = version,
+ description = description,
+ long_description = long_description,
+ long_description_content_type = "text/markdown",
+ author = author,
+ author_email = author_email,
+ url = url,
+ download_url = download_url,
+ license = license,
+ packages = setuptools.find_packages(),
+ include_package_data = True,
+ classifiers = [
+ "Development Status :: 3 - Alpha",
+ "Framework :: Jupyter",
+ "Intended Audience :: Science/Research",
+ "License :: OSI Approved :: MIT License",
+ "Natural Language :: English",
+ "Programming Language :: Python :: 3.6",
+ "Operating System :: OS Independent"],
+ install_requires = [
+ "ipython>=5.3.0",
+ "notebook>=5.0.0",
+ "ipykernel>=4.6.1",
+ "pandas>=0.20.1",
+ "numpy",
+ "adal",
+ "azure-mgmt-resource",
+ "azure-mgmt-datalake-analytics",
+ "azure-mgmt-datalake-store",
+ "azure-datalake-store"])