From 0e90d43ee2df9b03b88e30fcec094d94d984d35b Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Thu, 25 Jul 2024 16:20:01 -0600 Subject: [PATCH 01/15] first pass at python packaging and distribution --- .../python/packaging_and_distribution.md | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 docs/source/programming_languages/python/packaging_and_distribution.md diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md new file mode 100644 index 0000000..4a31f19 --- /dev/null +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -0,0 +1,206 @@ +> :warning: **Warning:** More information is needed to complete this guideline + +# Python Packaging and Distribution +> **Warning** Need to add a short description of the guideline + +## Purpose for this guideline +> **Warning** Need to add an explanation of how this guideline supports DS workflows, meets internal and external +> policies, and aids in collaboration and our overall success + +## Options and Applications for this guideline + +### PyPI (`pip install`) + +#### PyPI resources: + +- [PyPI Help Page](https://pypi.org/help/) + +- [Setting up a PyPI account](https://pypi.org/account/register/) + +- [Getting a PyPI access token](https://pypi.org/help/#apitoken) + +#### How to Apply this guideline +### Poetry Project Configuration Example – Library Package +``` +# pyproject.toml +# See: https://python-poetry.org/docs/pyproject/ + +[tool.poetry] +name = "my_python_package" +version = "0.1.0" +description = "Science data processing library and applications for some instrument." +authors = [ # Alphabetical + "Jane Doe ", + "John Doe " +] + +# Configure private PyPI repo to download packages +[[tool.poetry.source]] +name = "lasp-pypi" # This name will be used in the configuration to retrieve the proper credentials +url = "https://artifacts.pdmz.lasp.colorado.edu/repository/lasp-pypi/simple" # URL used to download your private packages + +# Dependency specification for core package +[tool.poetry.dependencies] +python = "^3.9" +astropy = "^4.2.1" +h5py = "^3.3.0" +numpy = "^1.21.0" +spiceypy = "^4.0.1" +lasp-packets = "1.2" +requests = "^2.26.0" +SQLAlchemy = "^1.4.27" +psycopg2 = "^2.9.2" +cloudpathlib = {extras = ["s3"], version = "^0.6.2"} + +# Development dependencies +[tool.poetry.dev-dependencies] +pytest-cov = "^2.12.1" +pylint = "^2.9.3" +responses = "^0.14.0" +pytest-randomly = "^3.10.2" +moto = {extras = ["s3"], version = "^2.2.16"} + +# Script entrypoints to put in installed bin directory +[tool.poetry.scripts] +sdp = 'my_python_package.cli:main' + +# Poetry boilerplate +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" +``` + +### Built-In (build + twine) + +#### Built-In resources + +- [Python Packaging User Guide](https://packaging.python.org/en/latest/): + +#### How to Apply this guideline +The link below is a pretty complete tutorial. There are also instructions for using +various other build tools. + +https://packaging.python.org/en/latest/tutorials/packaging-projects/ + +### Poetry + +[Poetry Build and Publish Docs](https://python-poetry.org/docs/cli/#build): + +poetry lock +poetry install +poetry version +poetry build +PYPI_USERNAME=__token__ +PYPI_TOKEN= +poetry publish # You will be prompted for your PyPI credentials if you don't provide the environment variables + +### Packaging for Conda (`conda install`) +Conda + +> **Need Help:** Anyone want to volunteer to write this section? TAG + +## How to apply this guideline +Detailed instructions or general guidance for implementation of the guideline +## Examples + +### Setuptools Example – Library Package +``` +""" +Setup file for the science data processing pipeline. + +The only required fields for setup are name, version, and packages. Other fields to consider (from looking at other +projects): keywords, include_package_data, requires, tests_require, package_data +""" +from setuptools import setup, find_packages + +# Reads the requirements file +with open('requirements.txt') as f: + requirements = f.read().splitlines() + +setup( + name='my_py_library', + version='0.1.0', + author='Jane Doe, John Doe, This is just a str', + author_email='jane.doe@lasp.colorado.edu', + description='Science data processing pipeline for the instrument', + long_description=open('README.md', 'r').read(), # Reads the readme file + python_requires='>=3.8, <4', + url='https://some-git.url', + classifiers=[ + "Natural Language :: English", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Astronomy", + "Programming Language :: Python :: 3.8", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + ], + packages=find_packages(exclude=('tests', 'tests.*')), + package_data={ + "my_py_library": [ + "some_necessary_config_data.json", + "calibration_data/*" + ] + }, + py_modules=['root_level_module_name',], + install_requires=requirements, + entry_points={ + 'console_scripts': [ + 'run-processing=my_py_library.cli:main', # package.module:function + ] + } +) +``` + +### Conda Package + +> :warning: Need a volunteer. +https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html + +## Useful Links +Here are some helpful resources: + +- [Python Packaging User's Guide](https://packaging.python.org/en/latest/) +- [The Hitchhiker's Guide to Python - Packaging your Code](https://docs.python-guide.org/shipping/packaging/) +- [The Sheer Joy of Packaging](https://python-packaging-tutorial.readthedocs.io/en/latest/index.html) +- [Package Python Projects the Proper Way with Poetry](https://hackersandslackers.com/python-poetry-package-manager/) +- [Poetry Documentation](https://python-poetry.org/docs/) +- [Setuptools Documentation](https://setuptools.pypa.io/en/latest/) +- [Building conda packages from scratch](https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html) + + + +## Some Terminology +### Library Packages vs Applications vs Scripts + +Python is a highly flexible interpreted language. This means it's easy to get started, quick to write, and easy to run. +Unfortunately, this also means it's very easy to write code that works in one context but not in others or code that is +robust but isn't designed to be inherited. + +Library Package: A python package, intended for redistribution, containing objects that serve as building blocks for +other developers to use. Examples are `numpy`, `pytest`, and `sqlalchemy`. + +Application: A python project (which may or may not be a packaged distribution) that provides specific and possibly +configurable functionality. Examples are Poetry, the AWS CLI, the Conda CLI, the Green Unicorn WSGI HTTP server, any +Django "app". + +Script: Pretty much anything else written in Python. One could arguably say that a Script is just a trivial Application +with little configuration or portability. Scripts are usually run with `python my_script.py argv`. They tend to be +difficult to maintain, update, or distribute. + +> **Note** A Python "package" is any directory containing an __init__.py file. + +Packaging Tooling: Not only managing a local environment, but also providing tooling for developing, building, and +distributing python packages for other users. While Conda does support this use case (it's how one creates and +distributes conda packages to conda-forge), Poetry and setuptools are much easier to develop with (for PyPI) and Poetry +boasts a similar dependency resolver to Conda. One major drawback to Conda in packaging is that there is no notion of an +"editable install" so the developer is forced to build and test end user functionality (e.g. script entrypoints) over +and over instead of simply making code changes in place. + +> :warning: Conda Develop: +> There is a conda subcommand called conda develop but it is not maintained and has a lot of issues. The maintainers of +conda recommend using pip install to install an editable package in development mode. + +See: https://github.com/conda/conda-build/issues/1992 + + +Original Author: Gavin Medley \ No newline at end of file From e06496bd34410375b7432feca6613674e5da3f00 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 28 Aug 2024 15:07:49 -0600 Subject: [PATCH 02/15] fix formatting --- .../programming_languages/python/packaging_and_distribution.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md index 4a31f19..2a0ae5c 100644 --- a/docs/source/programming_languages/python/packaging_and_distribution.md +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -19,7 +19,7 @@ - [Getting a PyPI access token](https://pypi.org/help/#apitoken) -#### How to Apply this guideline +## How to Apply this guideline ### Poetry Project Configuration Example – Library Package ``` # pyproject.toml From cbf4cf37903be4e2aac01fe367925f49ae21dc96 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 28 Aug 2024 15:12:48 -0600 Subject: [PATCH 03/15] fix email addresses for commit --- .../python/packaging_and_distribution.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md index 2a0ae5c..6bc9e46 100644 --- a/docs/source/programming_languages/python/packaging_and_distribution.md +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -30,8 +30,8 @@ name = "my_python_package" version = "0.1.0" description = "Science data processing library and applications for some instrument." authors = [ # Alphabetical - "Jane Doe ", - "John Doe " + "Jane Doe ", + "John Doe " ] # Configure private PyPI repo to download packages @@ -121,7 +121,7 @@ setup( name='my_py_library', version='0.1.0', author='Jane Doe, John Doe, This is just a str', - author_email='jane.doe@lasp.colorado.edu', + author_email='jane.doeATlasp.colorado.edu', description='Science data processing pipeline for the instrument', long_description=open('README.md', 'r').read(), # Reads the readme file python_requires='>=3.8, <4', From ec24c580a6a91b81e798f17ff8e55cce6454fddc Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 28 Aug 2024 15:14:42 -0600 Subject: [PATCH 04/15] don't worry about it --- .../python/packaging_and_distribution.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md index 6bc9e46..2a0ae5c 100644 --- a/docs/source/programming_languages/python/packaging_and_distribution.md +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -30,8 +30,8 @@ name = "my_python_package" version = "0.1.0" description = "Science data processing library and applications for some instrument." authors = [ # Alphabetical - "Jane Doe ", - "John Doe " + "Jane Doe ", + "John Doe " ] # Configure private PyPI repo to download packages @@ -121,7 +121,7 @@ setup( name='my_py_library', version='0.1.0', author='Jane Doe, John Doe, This is just a str', - author_email='jane.doeATlasp.colorado.edu', + author_email='jane.doe@lasp.colorado.edu', description='Science data processing pipeline for the instrument', long_description=open('README.md', 'r').read(), # Reads the readme file python_requires='>=3.8, <4', From da15536143f7bb62e523c4db0c8315276fa082c3 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 28 Aug 2024 15:23:17 -0600 Subject: [PATCH 05/15] add index.rst files --- docs/source/index.rst | 1 + docs/source/programming_languages/index.rst | 7 +++++++ docs/source/programming_languages/python/index.rst | 7 +++++++ 3 files changed, 15 insertions(+) create mode 100644 docs/source/programming_languages/index.rst create mode 100644 docs/source/programming_languages/python/index.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 0203a47..74d7b22 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,3 +7,4 @@ Welcome to the LASP Developer's Guide! :maxdepth: 1 licensing + programming_languages/index diff --git a/docs/source/programming_languages/index.rst b/docs/source/programming_languages/index.rst new file mode 100644 index 0000000..8187821 --- /dev/null +++ b/docs/source/programming_languages/index.rst @@ -0,0 +1,7 @@ +Programming Languages +===================== + +..toctree:: + :maxdepth: + + python/index diff --git a/docs/source/programming_languages/python/index.rst b/docs/source/programming_languages/python/index.rst new file mode 100644 index 0000000..312944a --- /dev/null +++ b/docs/source/programming_languages/python/index.rst @@ -0,0 +1,7 @@ +Python +===================== + +..toctree:: + :maxdepth: + + packaging_and_distbrution From cce5c1c6e4b4661a87bd46ede1b87e5be1cfc008 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 28 Aug 2024 15:26:01 -0600 Subject: [PATCH 06/15] add max depth --- docs/source/programming_languages/index.rst | 2 +- docs/source/programming_languages/python/index.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/programming_languages/index.rst b/docs/source/programming_languages/index.rst index 8187821..a91d28a 100644 --- a/docs/source/programming_languages/index.rst +++ b/docs/source/programming_languages/index.rst @@ -2,6 +2,6 @@ Programming Languages ===================== ..toctree:: - :maxdepth: + :maxdepth: 1 python/index diff --git a/docs/source/programming_languages/python/index.rst b/docs/source/programming_languages/python/index.rst index 312944a..13a6966 100644 --- a/docs/source/programming_languages/python/index.rst +++ b/docs/source/programming_languages/python/index.rst @@ -2,6 +2,6 @@ Python ===================== ..toctree:: - :maxdepth: + :maxdepth: 1 - packaging_and_distbrution + packaging_and_distribution From 28dcfc5a840662b1966456c894bb2e0357519966 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 28 Aug 2024 15:29:02 -0600 Subject: [PATCH 07/15] appropriate spacing --- docs/source/index.rst | 2 +- docs/source/programming_languages/index.rst | 4 ++-- docs/source/programming_languages/python/index.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 74d7b22..35577c0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,4 +7,4 @@ Welcome to the LASP Developer's Guide! :maxdepth: 1 licensing - programming_languages/index + programming_languages/index diff --git a/docs/source/programming_languages/index.rst b/docs/source/programming_languages/index.rst index a91d28a..e15d32c 100644 --- a/docs/source/programming_languages/index.rst +++ b/docs/source/programming_languages/index.rst @@ -1,7 +1,7 @@ Programming Languages ===================== -..toctree:: - :maxdepth: 1 +.. toctree:: + :maxdepth: 1 python/index diff --git a/docs/source/programming_languages/python/index.rst b/docs/source/programming_languages/python/index.rst index 13a6966..059a34d 100644 --- a/docs/source/programming_languages/python/index.rst +++ b/docs/source/programming_languages/python/index.rst @@ -1,7 +1,7 @@ Python ===================== -..toctree:: - :maxdepth: 1 +.. toctree:: + :maxdepth: 1 packaging_and_distribution From 7cca8f644dbb1b94e7c7425429c63272078b8d0a Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 28 Aug 2024 15:31:41 -0600 Subject: [PATCH 08/15] 3 spaces is the right amount of space --- docs/source/programming_languages/index.rst | 2 +- docs/source/programming_languages/python/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/programming_languages/index.rst b/docs/source/programming_languages/index.rst index e15d32c..f05b170 100644 --- a/docs/source/programming_languages/index.rst +++ b/docs/source/programming_languages/index.rst @@ -4,4 +4,4 @@ Programming Languages .. toctree:: :maxdepth: 1 - python/index + python/index diff --git a/docs/source/programming_languages/python/index.rst b/docs/source/programming_languages/python/index.rst index 059a34d..76b0cda 100644 --- a/docs/source/programming_languages/python/index.rst +++ b/docs/source/programming_languages/python/index.rst @@ -4,4 +4,4 @@ Python .. toctree:: :maxdepth: 1 - packaging_and_distribution + packaging_and_distribution From ccff229517140d57a0394344659f7e6501e2693d Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 28 Aug 2024 16:41:04 -0600 Subject: [PATCH 09/15] pull out the terminology section to its own document --- .../programming_languages/python/index.rst | 1 + .../python/packaging_and_distribution.md | 113 ++++++++---------- .../python/terminology.md | 40 +++++++ guideline_template.md | 4 +- 4 files changed, 91 insertions(+), 67 deletions(-) create mode 100644 docs/source/programming_languages/python/terminology.md diff --git a/docs/source/programming_languages/python/index.rst b/docs/source/programming_languages/python/index.rst index 76b0cda..932cbd3 100644 --- a/docs/source/programming_languages/python/index.rst +++ b/docs/source/programming_languages/python/index.rst @@ -4,4 +4,5 @@ Python .. toctree:: :maxdepth: 1 + terminology packaging_and_distribution diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md index 2a0ae5c..91ada7e 100644 --- a/docs/source/programming_languages/python/packaging_and_distribution.md +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -1,13 +1,18 @@ -> :warning: **Warning:** More information is needed to complete this guideline +> **Warning:** More information is needed to complete this guideline # Python Packaging and Distribution > **Warning** Need to add a short description of the guideline -## Purpose for this guideline +## Purpose > **Warning** Need to add an explanation of how this guideline supports DS workflows, meets internal and external > policies, and aids in collaboration and our overall success -## Options and Applications for this guideline +## Options +The options for Python packaging and distribution that we often see used at LASP are: +- [PyPI](#pypi--pip-install-) +- [Built-In](#built-in--build--twine-) +- [Poetry](#poetry) +- [Conda](#conda--conda-install-) ### PyPI (`pip install`) @@ -19,8 +24,38 @@ - [Getting a PyPI access token](https://pypi.org/help/#apitoken) -## How to Apply this guideline -### Poetry Project Configuration Example – Library Package +#### How to Use PyPI +> **Warning** Need to add + + +### Built-In (`build` + `twine`) + +> **Warning**: Need to add introductory paragraph that summarizes Built-In + +#### Built-In resources + +- [Python Packaging User Guide](https://packaging.python.org/en/latest/) + +#### How to use Built-In +The link below is a fairly complete tutorial. There are also instructions there for using various other build tools. +https://packaging.python.org/en/latest/tutorials/packaging-projects/ + +### Poetry + +> **Warning**: Need to add introductory paragraph that summarizes Built-In + +[Poetry Build and Publish Docs](https://python-poetry.org/docs/cli/#build): +``` +poetry lock +poetry install +poetry version +poetry build +PYPI_USERNAME=__token__ +PYPI_TOKEN= +poetry publish # You will be prompted for your PyPI credentials if you don't provide the environment variables +``` + +#### Poetry Project Configuration Example – Library Package ``` # pyproject.toml # See: https://python-poetry.org/docs/pyproject/ @@ -70,37 +105,12 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" ``` -### Built-In (build + twine) - -#### Built-In resources - -- [Python Packaging User Guide](https://packaging.python.org/en/latest/): - -#### How to Apply this guideline -The link below is a pretty complete tutorial. There are also instructions for using -various other build tools. - -https://packaging.python.org/en/latest/tutorials/packaging-projects/ - -### Poetry - -[Poetry Build and Publish Docs](https://python-poetry.org/docs/cli/#build): - -poetry lock -poetry install -poetry version -poetry build -PYPI_USERNAME=__token__ -PYPI_TOKEN= -poetry publish # You will be prompted for your PyPI credentials if you don't provide the environment variables - -### Packaging for Conda (`conda install`) -Conda - +### Conda (`conda install`) > **Need Help:** Anyone want to volunteer to write this section? TAG ## How to apply this guideline Detailed instructions or general guidance for implementation of the guideline + ## Examples ### Setuptools Example – Library Package @@ -153,7 +163,7 @@ setup( ### Conda Package -> :warning: Need a volunteer. +> Need a volunteer. https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html ## Useful Links @@ -167,40 +177,11 @@ Here are some helpful resources: - [Setuptools Documentation](https://setuptools.pypa.io/en/latest/) - [Building conda packages from scratch](https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html) - - -## Some Terminology -### Library Packages vs Applications vs Scripts - -Python is a highly flexible interpreted language. This means it's easy to get started, quick to write, and easy to run. -Unfortunately, this also means it's very easy to write code that works in one context but not in others or code that is -robust but isn't designed to be inherited. - -Library Package: A python package, intended for redistribution, containing objects that serve as building blocks for -other developers to use. Examples are `numpy`, `pytest`, and `sqlalchemy`. - -Application: A python project (which may or may not be a packaged distribution) that provides specific and possibly -configurable functionality. Examples are Poetry, the AWS CLI, the Conda CLI, the Green Unicorn WSGI HTTP server, any -Django "app". - -Script: Pretty much anything else written in Python. One could arguably say that a Script is just a trivial Application -with little configuration or portability. Scripts are usually run with `python my_script.py argv`. They tend to be -difficult to maintain, update, or distribute. - -> **Note** A Python "package" is any directory containing an __init__.py file. - -Packaging Tooling: Not only managing a local environment, but also providing tooling for developing, building, and -distributing python packages for other users. While Conda does support this use case (it's how one creates and -distributes conda packages to conda-forge), Poetry and setuptools are much easier to develop with (for PyPI) and Poetry -boasts a similar dependency resolver to Conda. One major drawback to Conda in packaging is that there is no notion of an -"editable install" so the developer is forced to build and test end user functionality (e.g. script entrypoints) over -and over instead of simply making code changes in place. - -> :warning: Conda Develop: -> There is a conda subcommand called conda develop but it is not maintained and has a lot of issues. The maintainers of -conda recommend using pip install to install an editable package in development mode. +> Conda Develop: +> There is a conda subcommand called `conda develop`, but it is not actively maintained. The maintainers of +conda recommend using `pip install` to install an editable package in development mode. See: https://github.com/conda/conda-build/issues/1992 -Original Author: Gavin Medley \ No newline at end of file +Credit: Content taken from a Confluence page originally written by Gavin Medley \ No newline at end of file diff --git a/docs/source/programming_languages/python/terminology.md b/docs/source/programming_languages/python/terminology.md new file mode 100644 index 0000000..f471823 --- /dev/null +++ b/docs/source/programming_languages/python/terminology.md @@ -0,0 +1,40 @@ +## Some Terminology + +Provide explanations of Python terminology that a user might encounter. + + +## Purpose for this guideline +Like all programming languages, Python has some terminology that is unique to it and it is helpful to have that language +explained. This page may be updated over time so that it holds the most useful terminology to those that use this +developer's guide. + +### Library Packages vs Applications vs Scripts + +Python is a highly flexible interpreted language. This means it's easy to get started, quick to write, and easy to run. +Unfortunately, this also means it's very easy to write code that works in one context but not in others or code that is +robust but isn't designed to be inherited. + +**Library Package**: A python package, intended for redistribution, containing objects that serve as building blocks for +other developers to use. Examples are `numpy`, `pytest`, and `sqlalchemy`. + +**Application**: A python project (which may or may not be a packaged distribution) that provides specific and possibly +configurable functionality. Examples are Poetry, the AWS CLI, the Conda CLI, the Green Unicorn WSGI HTTP server, any +Django "app". + +**Script**: Pretty much anything else written in Python. One could arguably say that a Script is just a trivial Application +with little configuration or portability. Scripts are usually run with `python my_script.py argv`. They tend to be +difficult to maintain, update, or distribute. + +> **Note** A Python "package" is any directory containing an `__init__.py` file. + +Packaging Tooling: Not only managing a local environment, but also providing tooling for developing, building, and +distributing python packages for other users. While Conda does support this use case (it's how one creates and +distributes conda packages to `conda-forge`), Poetry and `setuptools` are much easier to develop with (for PyPI) and Poetry +boasts a similar dependency resolver to Conda. One major drawback to Conda in packaging is that there is no notion of an +"editable install" so the developer is forced to build and test end user functionality (e.g. script entrypoints) over +and over instead of simply making code changes in place. + +## Useful Links +Helpful links to additional resources on the topic + +Credit: Content taken from a Confluence page originally written by Gavin Medley \ No newline at end of file diff --git a/guideline_template.md b/guideline_template.md index c5f567d..e971d9a 100644 --- a/guideline_template.md +++ b/guideline_template.md @@ -13,4 +13,6 @@ An explanation of the options available for this guideline (could be one or more Detailed instructions or general guidance for implementation of the guideline ## Useful Links -Helpful links to additional resources on the topic \ No newline at end of file +Helpful links to additional resources on the topic + +Credit: Content taken from a Confluence page originally written by xxxxxxx \ No newline at end of file From cd483c9824490726e19197104c6321f8302f0a6c Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Thu, 29 Aug 2024 08:54:27 -0600 Subject: [PATCH 10/15] fixing headings --- docs/source/programming_languages/python/terminology.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/programming_languages/python/terminology.md b/docs/source/programming_languages/python/terminology.md index f471823..fcfd504 100644 --- a/docs/source/programming_languages/python/terminology.md +++ b/docs/source/programming_languages/python/terminology.md @@ -1,8 +1,7 @@ -## Some Terminology +# Python Terminology Provide explanations of Python terminology that a user might encounter. - ## Purpose for this guideline Like all programming languages, Python has some terminology that is unique to it and it is helpful to have that language explained. This page may be updated over time so that it holds the most useful terminology to those that use this From 7060bc6700f36a77a512829b38dac0ee4d0edf25 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Thu, 29 Aug 2024 10:33:16 -0600 Subject: [PATCH 11/15] Updated formatting and organization --- .../programming_languages/python/terminology.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/programming_languages/python/terminology.md b/docs/source/programming_languages/python/terminology.md index fcfd504..9fece2f 100644 --- a/docs/source/programming_languages/python/terminology.md +++ b/docs/source/programming_languages/python/terminology.md @@ -1,8 +1,8 @@ # Python Terminology -Provide explanations of Python terminology that a user might encounter. +Some Python terminology that a user might encounter, particularly when working through this Python guide. -## Purpose for this guideline +## Purpose Like all programming languages, Python has some terminology that is unique to it and it is helpful to have that language explained. This page may be updated over time so that it holds the most useful terminology to those that use this developer's guide. @@ -13,9 +13,11 @@ Python is a highly flexible interpreted language. This means it's easy to get st Unfortunately, this also means it's very easy to write code that works in one context but not in others or code that is robust but isn't designed to be inherited. -**Library Package**: A python package, intended for redistribution, containing objects that serve as building blocks for +**Package**: A `python` package, intended for redistribution, containing objects that serve as building blocks for other developers to use. Examples are `numpy`, `pytest`, and `sqlalchemy`. +> **Note** A Python "package" is any directory containing an `__init__.py` file. + **Application**: A python project (which may or may not be a packaged distribution) that provides specific and possibly configurable functionality. Examples are Poetry, the AWS CLI, the Conda CLI, the Green Unicorn WSGI HTTP server, any Django "app". @@ -24,9 +26,7 @@ Django "app". with little configuration or portability. Scripts are usually run with `python my_script.py argv`. They tend to be difficult to maintain, update, or distribute. -> **Note** A Python "package" is any directory containing an `__init__.py` file. - -Packaging Tooling: Not only managing a local environment, but also providing tooling for developing, building, and +**Packaging Tooling**: Not only managing a local environment, but also providing tooling for developing, building, and distributing python packages for other users. While Conda does support this use case (it's how one creates and distributes conda packages to `conda-forge`), Poetry and `setuptools` are much easier to develop with (for PyPI) and Poetry boasts a similar dependency resolver to Conda. One major drawback to Conda in packaging is that there is no notion of an From b6eeafe0c2786564eb1e89b28096ea1eaa3e7261 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Thu, 29 Aug 2024 10:55:05 -0600 Subject: [PATCH 12/15] update formatting and logical progression --- .../python/packaging_and_distribution.md | 184 +++++++++--------- .../python/terminology.md | 5 +- guideline_template.md | 2 +- 3 files changed, 96 insertions(+), 95 deletions(-) diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md index 91ada7e..8ff719f 100644 --- a/docs/source/programming_languages/python/packaging_and_distribution.md +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -1,7 +1,7 @@ > **Warning:** More information is needed to complete this guideline # Python Packaging and Distribution -> **Warning** Need to add a short description of the guideline +Examples of Python packaging and distribution options and how use them. ## Purpose > **Warning** Need to add an explanation of how this guideline supports DS workflows, meets internal and external @@ -9,12 +9,10 @@ ## Options The options for Python packaging and distribution that we often see used at LASP are: -- [PyPI](#pypi--pip-install-) -- [Built-In](#built-in--build--twine-) -- [Poetry](#poetry) -- [Conda](#conda--conda-install-) +- [PyPI](#packaging-for-pypi--pip-install-) +- [Conda](#packaging-for-conda--conda-install-) -### PyPI (`pip install`) +### Packaging for PyPI (`pip install`) #### PyPI resources: @@ -24,96 +22,24 @@ The options for Python packaging and distribution that we often see used at LASP - [Getting a PyPI access token](https://pypi.org/help/#apitoken) -#### How to Use PyPI -> **Warning** Need to add - -### Built-In (`build` + `twine`) +#### Built-In (`build` + `twine`) > **Warning**: Need to add introductory paragraph that summarizes Built-In -#### Built-In resources - -- [Python Packaging User Guide](https://packaging.python.org/en/latest/) - -#### How to use Built-In -The link below is a fairly complete tutorial. There are also instructions there for using various other build tools. +##### How to use Built-In +Python Packaging User Guide: https://packaging.python.org/en/latest/ +The link below is a fairly complete tutorial. There are also instructions there for using various other build tools: https://packaging.python.org/en/latest/tutorials/packaging-projects/ -### Poetry - -> **Warning**: Need to add introductory paragraph that summarizes Built-In - -[Poetry Build and Publish Docs](https://python-poetry.org/docs/cli/#build): -``` -poetry lock -poetry install -poetry version -poetry build -PYPI_USERNAME=__token__ -PYPI_TOKEN= -poetry publish # You will be prompted for your PyPI credentials if you don't provide the environment variables -``` - -#### Poetry Project Configuration Example – Library Package -``` -# pyproject.toml -# See: https://python-poetry.org/docs/pyproject/ - -[tool.poetry] -name = "my_python_package" -version = "0.1.0" -description = "Science data processing library and applications for some instrument." -authors = [ # Alphabetical - "Jane Doe ", - "John Doe " -] - -# Configure private PyPI repo to download packages -[[tool.poetry.source]] -name = "lasp-pypi" # This name will be used in the configuration to retrieve the proper credentials -url = "https://artifacts.pdmz.lasp.colorado.edu/repository/lasp-pypi/simple" # URL used to download your private packages - -# Dependency specification for core package -[tool.poetry.dependencies] -python = "^3.9" -astropy = "^4.2.1" -h5py = "^3.3.0" -numpy = "^1.21.0" -spiceypy = "^4.0.1" -lasp-packets = "1.2" -requests = "^2.26.0" -SQLAlchemy = "^1.4.27" -psycopg2 = "^2.9.2" -cloudpathlib = {extras = ["s3"], version = "^0.6.2"} - -# Development dependencies -[tool.poetry.dev-dependencies] -pytest-cov = "^2.12.1" -pylint = "^2.9.3" -responses = "^0.14.0" -pytest-randomly = "^3.10.2" -moto = {extras = ["s3"], version = "^2.2.16"} - -# Script entrypoints to put in installed bin directory -[tool.poetry.scripts] -sdp = 'my_python_package.cli:main' - -# Poetry boilerplate -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" -``` - -### Conda (`conda install`) -> **Need Help:** Anyone want to volunteer to write this section? TAG +##### Built-In resources -## How to apply this guideline -Detailed instructions or general guidance for implementation of the guideline +- [Python Packaging User Guide](https://packaging.python.org/en/latest/) -## Examples +##### Setuptools Example – Library Package +
+ setup.py -### Setuptools Example – Library Package ``` """ Setup file for the science data processing pipeline. @@ -160,10 +86,85 @@ setup( } ) ``` +
+ + +#### Poetry -### Conda Package +> **Warning**: Need to add introductory paragraph that summarizes Built-In + +[Poetry Build and Publish Docs](https://python-poetry.org/docs/cli/#build) + +How to Publish to PyPI from Poetry +``` +poetry lock +poetry install +poetry version +poetry build +PYPI_USERNAME=__token__ +PYPI_TOKEN= +poetry publish # You will be prompted for your PyPI credentials if you don't provide the environment variables +``` -> Need a volunteer. +##### Poetry Project Configuration Example – Library Package + +
+ pyproject.toml + + ``` + # pyproject.toml + # See: https://python-poetry.org/docs/pyproject/ + + [tool.poetry] + name = "my_python_package" + version = "0.1.0" + description = "Science data processing library and applications for some instrument." + authors = [ # Alphabetical + "Jane Doe ", + "John Doe " + ] + + # Configure private PyPI repo to download packages + [[tool.poetry.source]] + name = "lasp-pypi" # This name will be used in the configuration to retrieve the proper credentials + url = "https://artifacts.pdmz.lasp.colorado.edu/repository/lasp-pypi/simple" # URL used to download your private packages + + # Dependency specification for core package + [tool.poetry.dependencies] + python = "^3.9" + astropy = "^4.2.1" + h5py = "^3.3.0" + numpy = "^1.21.0" + spiceypy = "^4.0.1" + lasp-packets = "1.2" + requests = "^2.26.0" + SQLAlchemy = "^1.4.27" + psycopg2 = "^2.9.2" + cloudpathlib = {extras = ["s3"], version = "^0.6.2"} + + # Development dependencies + [tool.poetry.dev-dependencies] + pytest-cov = "^2.12.1" + pylint = "^2.9.3" + responses = "^0.14.0" + pytest-randomly = "^3.10.2" + moto = {extras = ["s3"], version = "^2.2.16"} + + # Script entrypoints to put in installed bin directory + [tool.poetry.scripts] + sdp = 'my_python_package.cli:main' + + # Poetry boilerplate + [build-system] + requires = ["poetry-core>=1.0.0"] + build-backend = "poetry.core.masonry.api" + ``` +
+ +### Packaging for Conda (`conda install`) +> **Warning**: Need a volunteer + +## How to install and use Conda https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html ## Useful Links @@ -180,8 +181,7 @@ Here are some helpful resources: > Conda Develop: > There is a conda subcommand called `conda develop`, but it is not actively maintained. The maintainers of conda recommend using `pip install` to install an editable package in development mode. - -See: https://github.com/conda/conda-build/issues/1992 +> See: https://github.com/conda/conda-build/issues/1992 -Credit: Content taken from a Confluence page originally written by Gavin Medley \ No newline at end of file +Credit: Content taken from a Confluence guide written by Gavin Medley diff --git a/docs/source/programming_languages/python/terminology.md b/docs/source/programming_languages/python/terminology.md index 9fece2f..84937d3 100644 --- a/docs/source/programming_languages/python/terminology.md +++ b/docs/source/programming_languages/python/terminology.md @@ -16,7 +16,8 @@ robust but isn't designed to be inherited. **Package**: A `python` package, intended for redistribution, containing objects that serve as building blocks for other developers to use. Examples are `numpy`, `pytest`, and `sqlalchemy`. -> **Note** A Python "package" is any directory containing an `__init__.py` file. +> **Note** While most people think of a Python package as the above definition, a Python "package" is technically +> any directory containing an `__init__.py` file. **Application**: A python project (which may or may not be a packaged distribution) that provides specific and possibly configurable functionality. Examples are Poetry, the AWS CLI, the Conda CLI, the Green Unicorn WSGI HTTP server, any @@ -36,4 +37,4 @@ and over instead of simply making code changes in place. ## Useful Links Helpful links to additional resources on the topic -Credit: Content taken from a Confluence page originally written by Gavin Medley \ No newline at end of file +Credit: Content taken from a Confluence guide written by Gavin Medley diff --git a/guideline_template.md b/guideline_template.md index e971d9a..c99d67d 100644 --- a/guideline_template.md +++ b/guideline_template.md @@ -15,4 +15,4 @@ Detailed instructions or general guidance for implementation of the guideline ## Useful Links Helpful links to additional resources on the topic -Credit: Content taken from a Confluence page originally written by xxxxxxx \ No newline at end of file +Credit: Content taken from a Confluence guide written by From e027f837965fcfdd458229c63361d517f6a111a3 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Thu, 29 Aug 2024 10:58:19 -0600 Subject: [PATCH 13/15] make it make sense --- .../python/packaging_and_distribution.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md index 8ff719f..3cd0e95 100644 --- a/docs/source/programming_languages/python/packaging_and_distribution.md +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -167,6 +167,11 @@ poetry publish # You will be prompted for your PyPI credentials if you don't pr ## How to install and use Conda https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html +> Conda Develop: +> There is a conda subcommand called `conda develop`, but it is not actively maintained. The maintainers of +conda recommend using `pip install` to install an editable package in development mode. +> See: https://github.com/conda/conda-build/issues/1992 + ## Useful Links Here are some helpful resources: @@ -178,10 +183,4 @@ Here are some helpful resources: - [Setuptools Documentation](https://setuptools.pypa.io/en/latest/) - [Building conda packages from scratch](https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html) -> Conda Develop: -> There is a conda subcommand called `conda develop`, but it is not actively maintained. The maintainers of -conda recommend using `pip install` to install an editable package in development mode. -> See: https://github.com/conda/conda-build/issues/1992 - - Credit: Content taken from a Confluence guide written by Gavin Medley From 696e0ec753bc6da4e21b10e4859a437fe4f0fb96 Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Thu, 29 Aug 2024 11:01:33 -0600 Subject: [PATCH 14/15] Fix headings --- .../python/packaging_and_distribution.md | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md index 3cd0e95..40b92b8 100644 --- a/docs/source/programming_languages/python/packaging_and_distribution.md +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -12,9 +12,9 @@ The options for Python packaging and distribution that we often see used at LASP - [PyPI](#packaging-for-pypi--pip-install-) - [Conda](#packaging-for-conda--conda-install-) -### Packaging for PyPI (`pip install`) +## Packaging for PyPI (`pip install`) -#### PyPI resources: +### PyPI resources: - [PyPI Help Page](https://pypi.org/help/) @@ -23,20 +23,20 @@ The options for Python packaging and distribution that we often see used at LASP - [Getting a PyPI access token](https://pypi.org/help/#apitoken) -#### Built-In (`build` + `twine`) +### Built-In (`build` + `twine`) > **Warning**: Need to add introductory paragraph that summarizes Built-In -##### How to use Built-In +#### How to use Built-In Python Packaging User Guide: https://packaging.python.org/en/latest/ The link below is a fairly complete tutorial. There are also instructions there for using various other build tools: https://packaging.python.org/en/latest/tutorials/packaging-projects/ -##### Built-In resources +#### Built-In resources - [Python Packaging User Guide](https://packaging.python.org/en/latest/) -##### Setuptools Example – Library Package +#### Setuptools Example – Library Package
setup.py @@ -88,8 +88,7 @@ setup( ```
- -#### Poetry +### Poetry > **Warning**: Need to add introductory paragraph that summarizes Built-In @@ -106,7 +105,7 @@ PYPI_TOKEN= poetry publish # You will be prompted for your PyPI credentials if you don't provide the environment variables ``` -##### Poetry Project Configuration Example – Library Package +#### Poetry Project Configuration Example – Library Package
pyproject.toml @@ -161,10 +160,10 @@ poetry publish # You will be prompted for your PyPI credentials if you don't pr ```
-### Packaging for Conda (`conda install`) +## Packaging for Conda (`conda install`) > **Warning**: Need a volunteer -## How to install and use Conda +### How to install and use Conda https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html > Conda Develop: From d750436410ff06a353cbba8a8fb625c9e65b4b4c Mon Sep 17 00:00:00 2001 From: Keira Brooks Date: Wed, 18 Sep 2024 15:01:31 -0600 Subject: [PATCH 15/15] Update based on reviewer comments and add clarity --- .../python/packaging_and_distribution.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/programming_languages/python/packaging_and_distribution.md b/docs/source/programming_languages/python/packaging_and_distribution.md index 40b92b8..3a3d7b6 100644 --- a/docs/source/programming_languages/python/packaging_and_distribution.md +++ b/docs/source/programming_languages/python/packaging_and_distribution.md @@ -1,7 +1,7 @@ -> **Warning:** More information is needed to complete this guideline +> **Warning:** More information is needed to complete this guideline. # Python Packaging and Distribution -Examples of Python packaging and distribution options and how use them. +Examples of Python packaging and distribution options and how to use them. ## Purpose > **Warning** Need to add an explanation of how this guideline supports DS workflows, meets internal and external @@ -90,7 +90,7 @@ setup( ### Poetry -> **Warning**: Need to add introductory paragraph that summarizes Built-In +> **Warning**: Need to add introductory paragraph that summarizes Poetry [Poetry Build and Publish Docs](https://python-poetry.org/docs/cli/#build) @@ -161,7 +161,7 @@ poetry publish # You will be prompted for your PyPI credentials if you don't pr ## Packaging for Conda (`conda install`) -> **Warning**: Need a volunteer +> **Warning**: Need a volunteer to expand on Conda ### How to install and use Conda https://conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs.html