diff --git a/analytics_notebook/adlmagics/adlmagics/00_adlmagics_demo.ipynb b/adlmagics/00_adlmagics_demo.ipynb similarity index 99% rename from analytics_notebook/adlmagics/adlmagics/00_adlmagics_demo.ipynb rename to adlmagics/00_adlmagics_demo.ipynb index 9261c87..7883a19 100644 --- a/analytics_notebook/adlmagics/adlmagics/00_adlmagics_demo.ipynb +++ b/adlmagics/00_adlmagics_demo.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -825,7 +825,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.6.5" } }, "nbformat": 4, diff --git a/adlmagics/LICENSE b/adlmagics/LICENSE new file mode 100644 index 0000000..2107107 --- /dev/null +++ b/adlmagics/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/adlmagics/README.md b/adlmagics/README.md new file mode 100644 index 0000000..52585bc --- /dev/null +++ b/adlmagics/README.md @@ -0,0 +1,53 @@ + +# Azure Data Service Notebook +Azure Data Service Notebook is a set of tool for working with Azure Data Service (including HDInsight, Azure Data Lake, etc) + + +# Feature +Azure Data Service Notebook currently provides a set of magic commands for users to access Azure Data Lake. Available magics are captured in the table below. + +| Commands | Function | +|-----|-----| +|%adl login |Line magic to log in to Azure Data Lake.| +|%adl listaccounts|Line magic to list the Azure Data Lake analytic accounts for current user.| +|%adl listjobs|Line magic to list the Azure Data Lake jobs for a given account.| +|%%adl submitjob|Cell magic to submit a USQL job to ADL cluster.| +|%adl viewjob|Line magic to view job info.| +|%adl liststorefile|Line magic to list the Azure Data Lake store accounts.| +|%adl sample|Line magic to sample a given file, return results as Pandas DataFrame.| +|%adl logout|Line magic to log out.| + + +# Installation + +- Download and Install [python 3.6+](https://www.python.org/downloads/) +- Install jupyter: `pip install jupyter` +- Install azure packages fully : + `pip install azure` +- Or just install the key packages : + - `pip install adla` + - `pip install azure-mgmt-resource` + - `pip install azure-mgmt-datalake-analytics` + - `pip install azure-mgmt-datalake-store` + - `pip install azure-datalake-store` +- Copy the [adlmagic folder](/analytics_notebook/adlmagics) to local box. + + +# Examples +- [adlmagics_demo.ipynb](/analytics_notebook/adlmagics/00_adlmagics_demo.ipynb), demo file of Azure Data Lake job control functions. + + + +# Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/adlmagics/adlmagics/__init__.py b/adlmagics/adlmagics/__init__.py new file mode 100644 index 0000000..f71c7c5 --- /dev/null +++ b/adlmagics/adlmagics/__init__.py @@ -0,0 +1,4 @@ +from adlmagics.adlmagics_main import AdlMagics + +def load_ipython_extension(ipython): + ipython.register_magics(AdlMagics) \ No newline at end of file diff --git a/analytics_notebook/adlmagics/adlmagics/adlmagics.py b/adlmagics/adlmagics/adlmagics_main.py similarity index 68% rename from analytics_notebook/adlmagics/adlmagics/adlmagics.py rename to adlmagics/adlmagics/adlmagics_main.py index 7b13732..f083c83 100644 --- a/analytics_notebook/adlmagics/adlmagics/adlmagics.py +++ b/adlmagics/adlmagics/adlmagics_main.py @@ -2,25 +2,25 @@ from sys import stdout from os import linesep -from services.azure_token_service import AzureTokenService -from services.adla_service_sdk_impl import AdlaServiceSdkImpl -from services.adls_service_sdk_impl import AdlsServiceSdkImpl - -from magics.azure.azure_magic_base import AzureMagicBase -from magics.azure.azure_login_magic import AzureLoginMagic -from magics.azure.azure_logout_magic import AzureLogoutMagic - -from magics.adla.adla_magic_base import AdlaMagicBase -from magics.adla.adla_accounts_listing_magic import AdlaAccountsListingMagic -from magics.adla.adla_job_viewing_magic import AdlaJobViewingMagic -from magics.adla.adla_job_submission_magic import AdlaJobSubmissionMagic -from magics.adla.adla_jobs_listing_magic import AdlaJobsListingMagic - -from magics.adls.adls_magic_base import AdlsMagicBase -from magics.adls.adls_accounts_listing_magic import AdlsAccountsListingMagic -from magics.adls.adls_folders_listing_magic import AdlsFoldersListingMagic -from magics.adls.adls_files_listing_magic import AdlsFilesListingMagic -from magics.adls.adls_file_sampling_magic import AdlsFileSamplingMagic +from adlmagics.services.azure_token_service import AzureTokenService +from adlmagics.services.adla_service_sdk_impl import AdlaServiceSdkImpl +from adlmagics.services.adls_service_sdk_impl import AdlsServiceSdkImpl + +from adlmagics.magics.azure.azure_magic_base import AzureMagicBase +from adlmagics.magics.azure.azure_login_magic import AzureLoginMagic +from adlmagics.magics.azure.azure_logout_magic import AzureLogoutMagic + +from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase +from adlmagics.magics.adla.adla_accounts_listing_magic import AdlaAccountsListingMagic +from adlmagics.magics.adla.adla_job_viewing_magic import AdlaJobViewingMagic +from adlmagics.magics.adla.adla_job_submission_magic import AdlaJobSubmissionMagic +from adlmagics.magics.adla.adla_jobs_listing_magic import AdlaJobsListingMagic + +from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase +from adlmagics.magics.adls.adls_accounts_listing_magic import AdlsAccountsListingMagic +from adlmagics.magics.adls.adls_folders_listing_magic import AdlsFoldersListingMagic +from adlmagics.magics.adls.adls_files_listing_magic import AdlsFilesListingMagic +from adlmagics.magics.adls.adls_file_sampling_magic import AdlsFileSamplingMagic @magics_class class AdlMagics(Magics): @@ -88,7 +88,4 @@ def __register_adls_magic(self, adls_magic_class): self.__magics[adls_magic.cmd_name.lower()] = adls_magic def __write_line(self, text): - stdout.write(text + linesep) - -def load_ipython_extension(ipython): - ipython.register_magics(AdlMagics) \ No newline at end of file + stdout.write(text + linesep) \ No newline at end of file diff --git a/analytics_notebook/adlmagics/adlmagics/exceptions.py b/adlmagics/adlmagics/exceptions.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/exceptions.py rename to adlmagics/adlmagics/exceptions.py diff --git a/analytics_notebook/adlmagics/adlmagics/__init__.py b/adlmagics/adlmagics/magics/__init__.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/__init__.py rename to adlmagics/adlmagics/magics/__init__.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/__init__.py b/adlmagics/adlmagics/magics/adla/__init__.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/__init__.py rename to adlmagics/adlmagics/magics/adla/__init__.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py b/adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py similarity index 94% rename from analytics_notebook/adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py rename to adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py index 9c41bf7..71faf5e 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py +++ b/adlmagics/adlmagics/magics/adla/adla_accounts_listing_magic.py @@ -1,6 +1,6 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring -from magics.adla.adla_magic_base import AdlaMagicBase +from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase class AdlaAccountsListingMagic(AdlaMagicBase): def __init__(self, adla_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_consts.py b/adlmagics/adlmagics/magics/adla/adla_consts.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/adla/adla_consts.py rename to adlmagics/adlmagics/magics/adla/adla_consts.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py b/adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py similarity index 93% rename from analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py rename to adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py index 8b3b93a..19ccad5 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py +++ b/adlmagics/adlmagics/magics/adla/adla_job_submission_magic.py @@ -1,7 +1,7 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring -from magics.adla.adla_magic_base import AdlaMagicBase -from models.adla_job_submission import AdlaJobSubmission +from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase +from adlmagics.models.adla_job_submission import AdlaJobSubmission class AdlaJobSubmissionMagic(AdlaMagicBase): def __init__(self, adla_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py b/adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py similarity index 95% rename from analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py rename to adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py index f270bda..50624aa 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py +++ b/adlmagics/adlmagics/magics/adla/adla_job_viewing_magic.py @@ -1,6 +1,6 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring -from magics.adla.adla_magic_base import AdlaMagicBase +from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase class AdlaJobViewingMagic(AdlaMagicBase): def __init__(self, adla_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py b/adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py similarity index 97% rename from analytics_notebook/adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py rename to adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py index 39a9752..959de41 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py +++ b/adlmagics/adlmagics/magics/adla/adla_jobs_listing_magic.py @@ -1,7 +1,7 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring from os import linesep -from magics.adla.adla_magic_base import AdlaMagicBase +from adlmagics.magics.adla.adla_magic_base import AdlaMagicBase class AdlaJobsListingMagic(AdlaMagicBase): def __init__(self, adla_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_magic_base.py b/adlmagics/adlmagics/magics/adla/adla_magic_base.py similarity index 77% rename from analytics_notebook/adlmagics/adlmagics/magics/adla/adla_magic_base.py rename to adlmagics/adlmagics/magics/adla/adla_magic_base.py index d04162f..20a12d8 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adla/adla_magic_base.py +++ b/adlmagics/adlmagics/magics/adla/adla_magic_base.py @@ -1,4 +1,4 @@ -from magics.magic_base import MagicBase +from adlmagics.magics.magic_base import MagicBase class AdlaMagicBase(MagicBase): def __init__(self, cmd_name, adla_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adla/__init__.py b/adlmagics/adlmagics/magics/adls/__init__.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/adla/__init__.py rename to adlmagics/adlmagics/magics/adls/__init__.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py b/adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py similarity index 94% rename from analytics_notebook/adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py rename to adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py index 8af0832..c32cb98 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py +++ b/adlmagics/adlmagics/magics/adls/adls_accounts_listing_magic.py @@ -1,6 +1,6 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring -from magics.adls.adls_magic_base import AdlsMagicBase +from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase class AdlsAccountsListingMagic(AdlsMagicBase): def __init__(self, adls_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py b/adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py similarity index 95% rename from analytics_notebook/adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py rename to adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py index 50effee..2d3e1cc 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py +++ b/adlmagics/adlmagics/magics/adls/adls_file_sampling_magic.py @@ -1,7 +1,7 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring from pandas import DataFrame -from magics.adls.adls_magic_base import AdlsMagicBase +from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase class AdlsFileSamplingMagic(AdlsMagicBase): def __init__(self, adls_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py b/adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py similarity index 96% rename from analytics_notebook/adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py rename to adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py index 71fcd59..f114652 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py +++ b/adlmagics/adlmagics/magics/adls/adls_files_listing_magic.py @@ -1,6 +1,6 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring -from magics.adls.adls_magic_base import AdlsMagicBase +from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase class AdlsFilesListingMagic(AdlsMagicBase): def __init__(self, adls_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py b/adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py similarity index 96% rename from analytics_notebook/adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py rename to adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py index 4bb305e..546aa4f 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py +++ b/adlmagics/adlmagics/magics/adls/adls_folders_listing_magic.py @@ -1,6 +1,6 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring -from magics.adls.adls_magic_base import AdlsMagicBase +from adlmagics.magics.adls.adls_magic_base import AdlsMagicBase class AdlsFoldersListingMagic(AdlsMagicBase): def __init__(self, adls_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_magic_base.py b/adlmagics/adlmagics/magics/adls/adls_magic_base.py similarity index 77% rename from analytics_notebook/adlmagics/adlmagics/magics/adls/adls_magic_base.py rename to adlmagics/adlmagics/magics/adls/adls_magic_base.py index b576fe2..98f84d6 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/adls/adls_magic_base.py +++ b/adlmagics/adlmagics/magics/adls/adls_magic_base.py @@ -1,4 +1,4 @@ -from magics.magic_base import MagicBase +from adlmagics.magics.magic_base import MagicBase class AdlsMagicBase(MagicBase): def __init__(self, cmd_name, adls_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/adls/__init__.py b/adlmagics/adlmagics/magics/azure/__init__.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/adls/__init__.py rename to adlmagics/adlmagics/magics/azure/__init__.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/azure/azure_login_magic.py b/adlmagics/adlmagics/magics/azure/azure_login_magic.py similarity index 91% rename from analytics_notebook/adlmagics/adlmagics/magics/azure/azure_login_magic.py rename to adlmagics/adlmagics/magics/azure/azure_login_magic.py index 21343d0..1f6667b 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/azure/azure_login_magic.py +++ b/adlmagics/adlmagics/magics/azure/azure_login_magic.py @@ -1,6 +1,6 @@ from IPython.core.magic_arguments import magic_arguments, argument, parse_argstring -from magics.azure.azure_magic_base import AzureMagicBase +from adlmagics.magics.azure.azure_magic_base import AzureMagicBase class AzureLoginMagic(AzureMagicBase): def __init__(self, token_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/azure/azure_logout_magic.py b/adlmagics/adlmagics/magics/azure/azure_logout_magic.py similarity index 87% rename from analytics_notebook/adlmagics/adlmagics/magics/azure/azure_logout_magic.py rename to adlmagics/adlmagics/magics/azure/azure_logout_magic.py index 7c11325..2ab2180 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/azure/azure_logout_magic.py +++ b/adlmagics/adlmagics/magics/azure/azure_logout_magic.py @@ -1,4 +1,4 @@ -from magics.azure.azure_magic_base import AzureMagicBase +from adlmagics.magics.azure.azure_magic_base import AzureMagicBase class AzureLogoutMagic(AzureMagicBase): def __init__(self, token_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/azure/azure_magic_base.py b/adlmagics/adlmagics/magics/azure/azure_magic_base.py similarity index 78% rename from analytics_notebook/adlmagics/adlmagics/magics/azure/azure_magic_base.py rename to adlmagics/adlmagics/magics/azure/azure_magic_base.py index 04e28e2..2a406c3 100644 --- a/analytics_notebook/adlmagics/adlmagics/magics/azure/azure_magic_base.py +++ b/adlmagics/adlmagics/magics/azure/azure_magic_base.py @@ -1,4 +1,4 @@ -from magics.magic_base import MagicBase +from adlmagics.magics.magic_base import MagicBase class AzureMagicBase(MagicBase): def __init__(self, cmd_name, token_service): diff --git a/analytics_notebook/adlmagics/adlmagics/magics/magic_base.py b/adlmagics/adlmagics/magics/magic_base.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/magic_base.py rename to adlmagics/adlmagics/magics/magic_base.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/azure/__init__.py b/adlmagics/adlmagics/magics/session/__init__.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/azure/__init__.py rename to adlmagics/adlmagics/magics/session/__init__.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/session/session_list_magic.py b/adlmagics/adlmagics/magics/session/session_list_magic.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/session/session_list_magic.py rename to adlmagics/adlmagics/magics/session/session_list_magic.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/session/session_magic_base.py b/adlmagics/adlmagics/magics/session/session_magic_base.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/session/session_magic_base.py rename to adlmagics/adlmagics/magics/session/session_magic_base.py diff --git a/analytics_notebook/adlmagics/adlmagics/magics/session/__init__.py b/adlmagics/adlmagics/models/__init__.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/magics/session/__init__.py rename to adlmagics/adlmagics/models/__init__.py diff --git a/analytics_notebook/adlmagics/adlmagics/models/adla_account.py b/adlmagics/adlmagics/models/adla_account.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/models/adla_account.py rename to adlmagics/adlmagics/models/adla_account.py diff --git a/analytics_notebook/adlmagics/adlmagics/models/adla_job.py b/adlmagics/adlmagics/models/adla_job.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/models/adla_job.py rename to adlmagics/adlmagics/models/adla_job.py diff --git a/analytics_notebook/adlmagics/adlmagics/models/adla_job_submission.py b/adlmagics/adlmagics/models/adla_job_submission.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/models/adla_job_submission.py rename to adlmagics/adlmagics/models/adla_job_submission.py diff --git a/analytics_notebook/adlmagics/adlmagics/models/adls_account.py b/adlmagics/adlmagics/models/adls_account.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/models/adls_account.py rename to adlmagics/adlmagics/models/adls_account.py diff --git a/analytics_notebook/adlmagics/adlmagics/models/adls_file.py b/adlmagics/adlmagics/models/adls_file.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/models/adls_file.py rename to adlmagics/adlmagics/models/adls_file.py diff --git a/analytics_notebook/adlmagics/adlmagics/models/adls_folder.py b/adlmagics/adlmagics/models/adls_folder.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/models/adls_folder.py rename to adlmagics/adlmagics/models/adls_folder.py diff --git a/analytics_notebook/adlmagics/adlmagics/models/__init__.py b/adlmagics/adlmagics/services/__init__.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/models/__init__.py rename to adlmagics/adlmagics/services/__init__.py diff --git a/analytics_notebook/adlmagics/adlmagics/services/adla_service_sdk_impl.py b/adlmagics/adlmagics/services/adla_service_sdk_impl.py similarity index 94% rename from analytics_notebook/adlmagics/adlmagics/services/adla_service_sdk_impl.py rename to adlmagics/adlmagics/services/adla_service_sdk_impl.py index 29b2e9a..82d3d00 100644 --- a/analytics_notebook/adlmagics/adlmagics/services/adla_service_sdk_impl.py +++ b/adlmagics/adlmagics/services/adla_service_sdk_impl.py @@ -4,10 +4,10 @@ from azure.mgmt.datalake.analytics.job.models import JobInformation, USqlJobProperties, JobType from uuid import uuid4 -from exceptions import UserNotLoggedInError -from magics.adla.adla_consts import AdlaJobConsts -from models.adla_account import AdlaAccount -from models.adla_job import AdlaJob +from adlmagics.exceptions import UserNotLoggedInError +from adlmagics.magics.adla.adla_consts import AdlaJobConsts +from adlmagics.models.adla_account import AdlaAccount +from adlmagics.models.adla_job import AdlaJob class AdlaServiceSdkImpl: def __init__(self, token_service): diff --git a/analytics_notebook/adlmagics/adlmagics/services/adls_service_sdk_impl.py b/adlmagics/adlmagics/services/adls_service_sdk_impl.py similarity index 93% rename from analytics_notebook/adlmagics/adlmagics/services/adls_service_sdk_impl.py rename to adlmagics/adlmagics/services/adls_service_sdk_impl.py index 4d6d338..af07784 100644 --- a/analytics_notebook/adlmagics/adlmagics/services/adls_service_sdk_impl.py +++ b/adlmagics/adlmagics/services/adls_service_sdk_impl.py @@ -7,10 +7,10 @@ from datetime import datetime from sys import stdout -from exceptions import UserNotLoggedInError -from models.adls_account import AdlsAccount -from models.adls_folder import AdlsFolder -from models.adls_file import AdlsFile +from adlmagics.exceptions import UserNotLoggedInError +from adlmagics.models.adls_account import AdlsAccount +from adlmagics.models.adls_folder import AdlsFolder +from adlmagics.models.adls_file import AdlsFile class AdlsServiceSdkImpl: def __init__(self, token_service): @@ -62,7 +62,7 @@ def read_file(self, account, file_path, encoding, column_sep, to_be_read_line_co for read_line in f: read_lines.append(read_line.decode(encoding).strip().split(column_sep)) read_line_count += 1 - if (read_line_count > to_be_read_line_count): + if (read_line_count >= to_be_read_line_count): break return read_lines diff --git a/analytics_notebook/adlmagics/adlmagics/services/azure_token_service.py b/adlmagics/adlmagics/services/azure_token_service.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/services/azure_token_service.py rename to adlmagics/adlmagics/services/azure_token_service.py diff --git a/analytics_notebook/adlmagics/adlmagics/services/session_manager.py b/adlmagics/adlmagics/services/session_manager.py similarity index 100% rename from analytics_notebook/adlmagics/adlmagics/services/session_manager.py rename to adlmagics/adlmagics/services/session_manager.py diff --git a/adlmagics/requirements.txt b/adlmagics/requirements.txt new file mode 100644 index 0000000..11edbed --- /dev/null +++ b/adlmagics/requirements.txt @@ -0,0 +1,10 @@ +ipython>=5.3.0 +notebook>=5.0.0 +ipykernel>=4.6.1 +pandas>=0.20.1 +numpy +adal +azure-mgmt-resource +azure-mgmt-datalake-analytics +azure-mgmt-datalake-store +azure-datalake-store \ No newline at end of file diff --git a/adlmagics/setup.py b/adlmagics/setup.py new file mode 100644 index 0000000..9b43801 --- /dev/null +++ b/adlmagics/setup.py @@ -0,0 +1,54 @@ +import setuptools + +name = "adlmagics" +description = "Azure Data Lake management magics for Jupyter Notebook" +long_description = "" +version = "0.0.1" +packages = ["adlmagics", + "adlmagics/magics/adla", + "adlmagics/magics/adls", + "adlmagics/magics/azure", + "adlmagics/magics/session", + "adlmagics/models", + "adlmagics/services"] +author = "Vincent Feng" +author_email = "ivincentfeng@live.com" +url = "https://github.com/Azure/Azure-Data-Service-Notebook" +download_url = "https://github.com/Azure/Azure-Data-Service-Notebook" +license = "MIT" + +with open("README.md", "r") as f: + long_description = f.read() + +setuptools.setup( + name = name, + version = version, + description = description, + long_description = long_description, + long_description_content_type = "text/markdown", + author = author, + author_email = author_email, + url = url, + download_url = download_url, + license = license, + packages = setuptools.find_packages(), + include_package_data = True, + classifiers = [ + "Development Status :: 3 - Alpha", + "Framework :: Jupyter", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3.6", + "Operating System :: OS Independent"], + install_requires = [ + "ipython>=5.3.0", + "notebook>=5.0.0", + "ipykernel>=4.6.1", + "pandas>=0.20.1", + "numpy", + "adal", + "azure-mgmt-resource", + "azure-mgmt-datalake-analytics", + "azure-mgmt-datalake-store", + "azure-datalake-store"]) \ No newline at end of file diff --git a/analytics_notebook/adlmagics/adlmagics/services/__init__.py b/analytics_notebook/adlmagics/adlmagics/services/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/analytics_notebook/adlmagics/setup.py b/analytics_notebook/adlmagics/setup.py deleted file mode 100644 index fb83330..0000000 --- a/analytics_notebook/adlmagics/setup.py +++ /dev/null @@ -1,45 +0,0 @@ -DESCRIPTION = "Azure Data Lake management magics for Jupyter Notebook" -NAME = "adlmagics" -VERSION = "0.0.1" -PACKAGES = ["adlmagics", - "adlmagics/magics/adla", - "adlmagics/magics/adls", - "adlmagics/magics/azure", - "adlmagics/magics/session", - "adlmagics/models", - "adlmagics/services"] -AUTHOR = "" -AUTHOR_EMAIL = "" -URL = "https://github.com/Azure/Azure-Data-Service-Notebook" -DOWNLOAD_URL = "https://github.com/Azure/Azure-Data-Service-Notebook" -LICENSE = "MIT" - -from distutils.core import setup - -setup(name = NAME, - version = VERSION, - description = DESCRIPTION, - author = AUTHOR, - author_email = AUTHOR_EMAIL, - url = URL, - download_url = DOWNLOAD_URL, - license = LICENSE, - packages = PACKAGES, - include_package_data = True, - classifiers = [ - "Development Status :: 3 - Alpha", - "Framework :: Jupyter", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Natural Language :: English", - "Programming Language :: Python :: 3.6"], - install_requires = [ - "ipython>=5.3.0", - "notebook>=5.0.0", - "ipykernel>=4.6.1", - "pandas>=0.20.1", - "adal>=0.5.0", - "azure-mgmt-resource>=1.2.2", - "azure-mgmt-datalake-analytics>=0.3.0", - "azure-mgmt-datalake-store>=0.3.0", - "azure-datalake-store>=0.0.18"]) \ No newline at end of file