Merge pull request #824 from altendky/release/v0.5

.dockerignore

@@ -0,0 +1,5 @@
+.git
+.github
+.coveragerc
+.gitignore
+docker

.github/pull_request_template.md

@@ -1 +1,9 @@
-<!-- Please develop and target PRs against the development branch, not main. -->
+<!--
+Thanks for your interest in and effort contributing to plotman.
+
+Please develop and target PRs against the development branch, not main.
+
+Please read the contribution page in the wiki.
+
+https://github.com/ericaltendorf/plotman/wiki/Contributing
+-->

.github/workflows/ci.yml

@@ -107,10 +107,12 @@ jobs:
           - name: Test
             tox: test
             coverage: true
+          - name: Check hints
+            tox: check-hints
         include:
           - task:
-              name: Check
-              tox: check
+              name: Check manifest
+              tox: check-manifest
             os:
               name: Linux
               runs-on: ubuntu-latest

.gitignore

@@ -2,4 +2,5 @@ __pycache__
 venv
 .DS_Store
 .vscode
+*.code-workspace
 src/plotman.egg-info

CHANGELOG.md

@@ -5,14 +5,44 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5] - 2021-07-07
+### Fixed
+- `plotman kill` doesn't leave any temporary files behind anymore.
+  ([#801](https://github.com/ericaltendorf/plotman/pull/801))
+### Added
+- tmp directory overrides moved to `scheduling:` `tmp_overrides:`.
+  ([#758](https://github.com/ericaltendorf/plotman/pull/758))
+- Per tmp directory phase limit control added to `scheduling:` `tmp_overrides:`.
+  ([#758](https://github.com/ericaltendorf/plotman/pull/758))
+- `plotman export` command to output summaries from plot logs in `.csv` format.
+  ([#557](https://github.com/ericaltendorf/plotman/pull/557))
+- `--json` option for `plotman status`.
+  ([#549](https://github.com/ericaltendorf/plotman/pull/549))
+- If the tmp drive selected for a plot is also listed as a dst drive then plotman will use the same drive for both.
+  ([#643](https://github.com/ericaltendorf/plotman/pull/643))
+- `plotman prometheus` command to output status for consumption by [Prometheus](https://prometheus.io/).
+  ([#430](https://github.com/ericaltendorf/plotman/pull/430))
+- `plotman logs` command to print and tail plot logs by their plot ID.
+  ([#509](https://github.com/ericaltendorf/plotman/pull/509))
+- Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter).
+  See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up.
+  ([#797](https://github.com/ericaltendorf/plotman/pull/797))
+- Added argument `-f`/`--force` to `plotman kill` to skip confirmation before killing the job.
+  ([#801](https://github.com/ericaltendorf/plotman/pull/801))
+- Docker container support.
+  See the [docker configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Docker-Configuration) for help setting it up.
+  ([#783](https://github.com/ericaltendorf/plotman/pull/783))
+- Plot sizes other than k32 are handled.
+  ([#803](https://github.com/ericaltendorf/plotman/pull/803))
+
 ## [0.4.1] - 2021-06-11
 ### Fixed
 - Archival disk space check finds drives with multiple mount points again.
   This fixes a regression introduced in v0.4.1.
-  [#773](https://github.com/ericaltendorf/plotman/issues/773)
+  ([#773](https://github.com/ericaltendorf/plotman/issues/773))
 - `plotman dirs` does not fail for every invocation.
   `TypeError: dirs_report() missing 1 required positional argument: 'width'`
-  [#778](https://github.com/ericaltendorf/plotman/issues/778)
+  ([#778](https://github.com/ericaltendorf/plotman/issues/778))
 
 ## [0.4] - 2021-06-10
 ### Fixed

MANIFEST.in

@@ -3,8 +3,12 @@ include LICENSE*
 include README.md
 include *.md
 include VERSION
+include mypy.ini
 include tox.ini
 include .coveragerc
 recursive-include src *.py
 recursive-include src/plotman/_tests/resources *
 recursive-include src/plotman/resources *
+recursive-exclude docker *
+exclude .dockerignore
+exclude build-docker-plotman.sh

VERSION

@@ -1 +1 @@
-0.4.1
+0.5

build-docker-plotman.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+DOCKER_REGISTRY="<docker-registry>"
+PROJECT="chia-plotman"
+TAG="plotter"
+BASE_CONTAINER="ubuntu:20.04"
+CHIA_GIT_REFERENCE="1.1.7"
+
+# The UID/GID should match the 'chia' owner of the directories on the host system
+UID=10001
+GID=10001
+
+docker rmi ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
+
+docker build . \
+	--squash \
+	--build-arg BASE_CONTAINER=${BASE_CONTAINER} \
+	--build-arg CHIA_GIT_REFERENCE=${CHIA_GIT_REFERENCE} \
+	--build-arg UID=${UID} \
+	--build-arg GID=${GID} \
+	-f docker/Dockerfile \
+        -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
+
+docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG}

docker/Dockerfile

@@ -0,0 +1,56 @@
+# Build deployable artifacts
+ARG BASE_CONTAINER=ubuntu:20.04
+FROM ${BASE_CONTAINER} as plotman-builder
+
+ARG CHIA_GIT_REFERENCE
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 ansible tar bash ca-certificates git openssl unzip wget python3-pip sudo acl build-essential python3-dev python3.8-venv python3.8-distutils apt nfs-common python-is-python3
+
+RUN echo "cloning ${CHIA_GIT_REFERENCE}"
+RUN git clone --branch "${CHIA_GIT_REFERENCE}" https://github.com/Chia-Network/chia-blockchain.git \
+&& cd chia-blockchain \
+&& git submodule update --init mozilla-ca
+
+WORKDIR /chia-blockchain
+# Placeholder for patches
+RUN /bin/bash ./install.sh
+
+COPY . /plotman
+
+RUN ["/bin/bash", "-c", "source ./activate && pip install /plotman && deactivate"] 
+
+# Build deployment container
+FROM ${BASE_CONTAINER} as plotman
+
+ARG UID=10001
+ARG GID=10001
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+&& DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 python3.8-venv ca-certificates tzdata ssh rsync \
+&& apt-get clean all \
+&& rm -rf /var/lib/apt/lists
+
+COPY --from=plotman-builder /chia-blockchain /chia-blockchain
+
+RUN groupadd -g ${GID} chia
+RUN useradd -m -u ${UID} -g ${GID} chia
+
+RUN mkdir -p /data/chia/tmp \
+&& mkdir -p /data/chia/plots \
+&& mkdir -p /data/chia/logs
+
+VOLUME ["/data/chia/tmp","/data/chia/plots","/data/chia/logs"]
+
+RUN chown -R chia:chia /chia-blockchain \
+&& chown -R chia:chia /data/chia
+
+WORKDIR /chia-blockchain
+USER chia
+
+ENV VIRTUAL_ENV="/chia-blockchain/venv"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Kick off plots (assumes the environemnt is good to go)
+CMD ["/bin/bash", "-c", "plotman plot" ]
+# Alternative command to simply provide shell environment
+# CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ]

docker/sample.docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3"
+
+services:
+  chia_plotman:
+    restart: always
+    container_name: chia-plotman
+    image: ${DOCKER_IMAGE}
+    volumes:
+      - ${HOME}/.ssh:/home/chia/.ssh
+      - ${HOME}/.chia:/home/chia/.chia
+      - ${HOME}/.config:/home/chia/.config
+      - ${LOGS_DIR}:/data/chia/logs
+      - ${PLOTS_DIR}:/data/chia/plots
+      - ${PLOTS_TMP_DIR}:/data/chia/tmp
+      - /tmp:/tmp
+    logging:
+      options:
+        max-size: ${DOCKER_LOG_MAX_SIZE}
+        max-file: ${DOCKER_LOG_MAX_FILE}

docker/sample.env

@@ -0,0 +1,7 @@
+DOCKER_IMAGE=<docker-registry>/chia-plotman:plotter
+LOGS_DIR=/data/chia/logs
+PLOTS_DIR=/data/chia/plots
+PLOTS_TMP_DIR=/data/chia/tmp
+DOCKER_LOG_MAX_SIZE=4m
+DOCKER_LOG_MAX_FILE=10
+

mypy.ini

@@ -0,0 +1,26 @@
+[mypy]
+show_error_codes = true
+strict = true
+mypy_path = src/
+
+[mypy-appdirs]
+ignore_missing_imports = true
+
+[mypy-click]
+ignore_missing_imports = true
+
+[mypy-pendulum]
+# TODO: https://github.com/sdispater/pendulum/pull/551
+implicit_reexport = true
+
+[mypy-psutil]
+ignore_missing_imports = true
+
+[mypy-pyfakefs]
+ignore_missing_imports = true
+
+[mypy-texttable]
+ignore_missing_imports = true
+
+[mypy-yaml]
+ignore_missing_imports = true

setup.cfg

@@ -41,10 +41,12 @@ install_requires =
     click ~= 7.1
     desert == 2020.11.18
     marshmallow ~= 3.12
+    packaging ~= 20.9
     pendulum ~= 2.1
     psutil ~= 5.8
     pyyaml ~= 5.4
     texttable ~= 1.6
+    typing-extensions ~= 3.10
 
 [options.packages.find]
 where=src
@@ -62,10 +64,14 @@ dev =
     isort
 test =
     %(coverage)s
-    check-manifest
     pytest
     pytest-cov
     pyfakefs
+checks =
+    check-manifest ~= 0.46
+    mypy == 0.902
+    types-pkg_resources ~= 0.1.2
+    %(test)s
 
 [options.data_files]
 config = src/plotman/resources/plotman.yaml

src/plotman/_tests/archive_test.py

@@ -1,6 +1,6 @@
 from plotman import archive, job
 
 
-def test_compute_priority():
+def test_compute_priority() -> None:
     assert (archive.compute_priority( job.Phase(major=3, minor=1), 1000, 10) >
             archive.compute_priority( job.Phase(major=3, minor=6), 1000, 10) )

src/plotman/_tests/configuration_test.py

@@ -9,23 +9,23 @@
 
 
 @pytest.fixture(name='config_text')
-def config_text_fixture():
+def config_text_fixture() -> str:
     return importlib.resources.read_text(plotman_resources, "plotman.yaml")
 
 
 @pytest.fixture(name='target_definitions_text')
-def target_definitions_text_fixture():
+def target_definitions_text_fixture() -> str:
     return importlib.resources.read_text(
         plotman_resources, "target_definitions.yaml",
     )
 
 
-def test_get_validated_configs__default(config_text, target_definitions_text):
+def test_get_validated_configs__default(config_text: str, target_definitions_text: str) -> None:
     """Check that get_validated_configs() works with default/example plotman.yaml file."""
     res = configuration.get_validated_configs(config_text, '', target_definitions_text)
     assert isinstance(res, configuration.PlotmanConfig)
 
-def test_get_validated_configs__malformed(config_text, target_definitions_text):
+def test_get_validated_configs__malformed(config_text: str, target_definitions_text: str) -> None:
     """Check that get_validated_configs() raises exception with invalid plotman.yaml contents."""
     loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader)
 
@@ -39,7 +39,7 @@ def test_get_validated_configs__malformed(config_text, target_definitions_text):
     assert exc_info.value.args[0] == f"Config file at: '/the_path' is malformed"
 
 
-def test_get_validated_configs__missing():
+def test_get_validated_configs__missing() -> None:
     """Check that get_validated_configs() raises exception when plotman.yaml does not exist."""
     with pytest.raises(configuration.ConfigurationException) as exc_info:
         configuration.read_configuration_text('/invalid_path')
@@ -50,7 +50,7 @@ def test_get_validated_configs__missing():
     )
 
 
-def test_loads_without_user_interface(config_text, target_definitions_text):
+def test_loads_without_user_interface(config_text: str, target_definitions_text: str) -> None:
     loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader)
 
     del loaded_yaml["user_interface"]
@@ -62,7 +62,7 @@ def test_loads_without_user_interface(config_text, target_definitions_text):
     assert reloaded_yaml.user_interface == configuration.UserInterface()
 
 
-def test_loads_without_user_archiving(config_text, target_definitions_text):
+def test_loads_without_user_archiving(config_text: str, target_definitions_text: str) -> None:
     loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader)
 
     del loaded_yaml["archiving"]
@@ -74,30 +74,30 @@ def test_loads_without_user_archiving(config_text, target_definitions_text):
     assert reloaded_yaml.archiving is None
 
 
-def test_get_dst_directories_gets_dst():
+def test_get_dst_directories_gets_dst() -> None:
     tmp = ['/tmp']
     dst = ['/dst0', '/dst1']
     directories = configuration.Directories(tmp=tmp, dst=dst)
 
     assert directories.get_dst_directories() == dst
 
 
-def test_get_dst_directories_gets_tmp():
+def test_get_dst_directories_gets_tmp() -> None:
     tmp = ['/tmp']
     directories = configuration.Directories(tmp=tmp)
 
     assert directories.get_dst_directories() == tmp
 
 
-def test_dst_is_dst():
+def test_dst_is_dst() -> None:
     tmp = ['/tmp']
     dst = ['/dst0', '/dst1']
     directories = configuration.Directories(tmp=tmp, dst=dst)
 
     assert not directories.dst_is_tmp()
 
 
-def test_dst_is_tmp():
+def test_dst_is_tmp() -> None:
     tmp = ['/tmp']
     directories = configuration.Directories(tmp=tmp)