From 38523b3da77b542b7b2502c9474f37d05a56beb9 Mon Sep 17 00:00:00 2001 From: Florian Weyandt Date: Fri, 14 May 2021 12:42:42 +0200 Subject: [PATCH 001/109] add prometheus format status output --- src/plotman/_tests/reporting_test.py | 18 ++++++++++++ src/plotman/plotman.py | 8 +++++- src/plotman/reporting.py | 42 ++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/plotman/_tests/reporting_test.py b/src/plotman/_tests/reporting_test.py index 87c8a5e2..afc235f9 100644 --- a/src/plotman/_tests/reporting_test.py +++ b/src/plotman/_tests/reporting_test.py @@ -60,3 +60,21 @@ def test_job_viz_counts(): ] assert(reporting.job_viz(jobs) == '1 2 .:;! 3 ! 4 ') + +def test_to_prometheus_format(): + prom_stati = [ + ('foo="bar",baz="2"', {'metric1': 1, 'metric2': 2}), + ('foo="blubb",baz="3"', {'metric1': 2, 'metric2': 3}) + ] + metrics = {'metric1': 'This is foo', 'metric2': 'In a parallel universe this is foo'} + expected = [ + '# HELP metric1 This is foo.', + '# TYPE metric1 gauge', + 'metric1{foo="bar",baz="2"} 1', + 'metric1{foo="blubb",baz="3"} 2', + '# HELP metric2 In a parallel universe this is foo.', + '# TYPE metric2 gauge', + 'metric2{foo="bar",baz="2"} 2','metric2{foo="blubb",baz="3"} 3' + ] + result = reporting.to_prometheus_format(metrics, prom_stati) + assert(result == expected) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index d3468735..27fd4c60 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -27,7 +27,9 @@ def parse_args(self): sp.add_parser('version', help='print the version') sp.add_parser('status', help='show current plotting status') - + + sp.add_parser('prometheus', help='show current plotting status in prometheus readable format') + sp.add_parser('dirs', help='show directories info') sp.add_parser('interactive', help='run interactive control/monitoring mode') @@ -165,6 +167,10 @@ def main(): if args.cmd == 'status': print(reporting.status_report(jobs, get_term_width())) + # Prometheus report + if args.cmd == 'prometheus': + print(reporting.prometheus_report(jobs)) + # Directories report elif args.cmd == 'dirs': print(reporting.dirs_report(jobs, cfg.directories, cfg.scheduling, get_term_width())) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 3142b978..aa5ab987 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -125,6 +125,48 @@ def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''): # return ('tmp dir prefix: %s ; dst dir prefix: %s\n' % (tmp_prefix, dst_prefix) return tab.draw() +def to_prometheus_format(metrics, prom_stati): + prom_str_list = [] + for metric_name, metric_desc in metrics.items(): + prom_str_list.append(f'# HELP {metric_name} {metric_desc}.') + prom_str_list.append(f'# TYPE {metric_name} gauge') + for label_str, values in prom_stati: + prom_str_list.append('%s{%s} %s' % (metric_name, label_str, values[metric_name])) + return prom_str_list + +def prometheus_report(jobs, tmp_prefix='', dst_prefix=''): + metrics = { + 'plotman_plot_phase_major': 'The phase the plot is currently in', + 'plotman_plot_phase_minor': 'The part of the phase the plot is currently in', + 'plotman_plot_tmp_usage': 'Tmp dir usage in bytes', + 'plotman_plot_mem_usage': 'Memory usage in bytes', + 'plotman_plot_user_time': 'Processor time (user) in s', + 'plotman_plot_sys_time': 'Processor time (sys) in s', + 'plotman_plot_iowait_time': 'Processor time (iowait) in s', + } + prom_stati = [] + for j in jobs: + labels = { + 'plot_id': j.plot_id[:8], + 'tmp_dir': abbr_path(j.tmpdir, tmp_prefix), + 'dst_dir': abbr_path(j.dstdir, dst_prefix), + 'run_status': j.get_run_status(), + 'phase': phase_str(j.progress()) + } + label_str = ','.join([f'{k}="{v}"' for k, v in labels.items()]) + values = { + 'plotman_plot_phase_major': j.progress().major, + 'plotman_plot_phase_minor': j.progress().minor, + 'plotman_plot_tmp_usage': j.get_tmp_usage(), + 'plotman_plot_mem_usage': j.get_mem_usage(), + 'plotman_plot_user_time': j.get_time_user(), + 'plotman_plot_sys_time': j.get_time_sys(), + 'plotman_plot_iowait_time': j.get_time_iowait(), + } + prom_stati += [(label_str, values)] + return '\n'.join(to_prometheus_format(metrics, prom_stati)) + + def tmp_dir_report(jobs, dir_cfg, sched_cfg, width, start_row=None, end_row=None, prefix=''): '''start_row, end_row let you split the table up if you want''' tab = tt.Texttable() From b7715ef250cf57e538e6a9c35909808cf35530fc Mon Sep 17 00:00:00 2001 From: Rafael Steil Date: Mon, 17 May 2021 19:08:53 -0400 Subject: [PATCH 002/109] Add a log parser class --- src/plotman/_tests/log_parser_test.py | 54 +++++++ .../2021-04-04T19_00_47.681088-0400.log | 1 + src/plotman/log_parser.py | 140 ++++++++++++++++++ src/plotman/plotinfo.py | 108 ++++++++++++++ 4 files changed, 303 insertions(+) create mode 100644 src/plotman/_tests/log_parser_test.py create mode 100644 src/plotman/log_parser.py create mode 100644 src/plotman/plotinfo.py diff --git a/src/plotman/_tests/log_parser_test.py b/src/plotman/_tests/log_parser_test.py new file mode 100644 index 00000000..31e6c417 --- /dev/null +++ b/src/plotman/_tests/log_parser_test.py @@ -0,0 +1,54 @@ +import importlib.resources + +from plotman._tests import resources +from plotman.log_parser import PlotLogParser + +def test_should_correctly_parse(): + with importlib.resources.path(resources, "2021-04-04T19_00_47.681088-0400.log") as p: + logfile_path = p + + parser = PlotLogParser() + info = parser.parse(logfile_path) + + assert info.plot_id == "3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24" + + assert info.plot_size == 32 + assert info.started_at == "Sun Apr 4 19:00:50 2021" + assert info.buffer == 4000 + assert info.buckets == 128 + assert info.threads == 4 + + assert info.tmp_dir1 == "/farm/yards/901" + assert info.tmp_dir2 == "/farm/yards/901" + + assert info.phase1_duration_raw == 17571.981 + assert info.phase1_duration == 17572 + assert info.phase1_duration_minutes == 293 + assert info.phase1_duration_hours == 4.88 + + assert info.phase2_duration_raw == 6911.621 + assert info.phase2_duration == 6912 + assert info.phase2_duration_minutes == 115 + assert info.phase2_duration_hours == 1.92 + + assert info.phase3_duration_raw == 14537.188 + assert info.phase3_duration == 14537 + assert info.phase3_duration_minutes == 242 + assert info.phase3_duration_hours == 4.04 + + assert info.phase4_duration_raw == 924.288 + assert info.phase4_duration == 924 + assert info.phase4_duration_minutes == 15 + assert info.phase4_duration_hours == 0.26 + + assert info.total_time_raw == 39945.080 + assert info.total_time == 39945 + assert info.total_time_minutes == 666 + assert info.total_time_hours == 11.10 + + assert info.copy_time_raw == 501.696 + assert info.copy_time == 502 + assert info.copy_time_minutes == 8 + assert info.copy_time_hours == 0.14 + + assert info.filename == "/farm/wagons/801/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot" \ No newline at end of file diff --git a/src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.log b/src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.log index 704efe51..ce11a343 100644 --- a/src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.log +++ b/src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.log @@ -2081,6 +2081,7 @@ Approximate working space used (without final file): 286.598 GiB Final File size: 101.336 GiB Total time = 39945.080 seconds. CPU (123.100%) Mon Apr 5 06:06:35 2021 Copied final file from "/farm/yards/901/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot.2.tmp" to "/farm/wagons/801/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot.2.tmp" +Copy time = 501.696 seconds. CPU (23.860%) Sun May 9 22:52:41 2021 Removed temp2 file "/farm/yards/901/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot.2.tmp"? 1 Renamed final file from "/farm/wagons/801/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot.2.tmp" to "/farm/wagons/801/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot" 06:22:40.715 src.plotting.create_plots : INFO  Summary: diff --git a/src/plotman/log_parser.py b/src/plotman/log_parser.py new file mode 100644 index 00000000..5dbfcfe0 --- /dev/null +++ b/src/plotman/log_parser.py @@ -0,0 +1,140 @@ +import re +from plotman.plotinfo import PlotInfo + +class PlotLogParser: + """Parser for a finished plotting job""" + + def parse(self, filename: str) -> PlotInfo: + """Parses a single log and returns its info""" + entry = PlotInfo() + + matchers = [ + self.ignore_line, + self.plot_id, + self.plot_start_date, + self.plot_size, + self.buffer_size, + self.buckets, + self.threads, + self.plot_dirs, + self.phase1_duration, + self.phase2_duration, + self.phase3_duration, + self.phase4_duration, + self.total_time, + self.copy_time, + self.filename + ] + + with open(filename, 'r') as f: + for line in f: + for matcher in matchers: + if (matcher(line, entry)): + break + + return entry + + # ID: 3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24 + def plot_id(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r'^ID: (.+)$', line) + if m: + entry.plot_id = m.group(1) + return m != None + + # Renamed final file from "/farm/wagons/801/abc.plot.2.tmp" to "/farm/wagons/801/abc.plot" + def filename(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r'^Renamed final file from ".+" to "(.+)"', line) + if m: + entry.filename = m.group(1) + return m != None + + # Time for phase 1 = 17571.981 seconds. CPU (178.600%) Sun Apr 4 23:53:42 2021 + def phase1_duration(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Time for phase 1 = (\d+\.\d+) seconds", line) + if m: + entry.phase1_duration_raw = float(m.group(1)) + return m != None + + # Time for phase 2 = 6911.621 seconds. CPU (71.780%) Mon Apr 5 01:48:54 2021 + def phase2_duration(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Time for phase 2 = (\d+\.\d+) seconds", line) + if m: + entry.phase2_duration_raw = float(m.group(1)) + return m != None + + # Time for phase 3 = 14537.188 seconds. CPU (82.730%) Mon Apr 5 05:51:11 2021 + def phase3_duration(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Time for phase 3 = (\d+\.\d+) seconds", line) + if m: + entry.phase3_duration_raw = float(m.group(1)) + return m != None + + # Time for phase 4 = 924.288 seconds. CPU (86.810%) Mon Apr 5 06:06:35 2021 + def phase4_duration(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Time for phase 4 = (\d+\.\d+) seconds", line) + if m: + entry.phase4_duration_raw = float(m.group(1)) + return m != None + + # Total time = 39945.080 seconds. CPU (123.100%) Mon Apr 5 06:06:35 2021 + def total_time(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Total time = (\d+\.\d+) seconds", line) + if m: + entry.total_time_raw = float(m.group(1)) + return m != None + + # Copy time = 501.696 seconds. CPU (23.860%) Sun May 9 22:52:41 2021 + def copy_time(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Copy time = (\d+\.\d+) seconds", line) + if m: + entry.copy_time_raw = float(m.group(1)) + return m != None + + # Starting plotting progress into temporary dirs: /farm/yards/901 and /farm/yards/901 + def plot_dirs(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Starting plotting progress into temporary dirs: (.+) and (.+)$", line) + if m: + entry.tmp_dir1 = m.group(1) + entry.tmp_dir2 = m.group(2) + return m != None + + # Using 4 threads of stripe size 65536 + def threads(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Using (\d+) threads of stripe size (\d+)", line) + if m: + entry.threads = int(m.group(1)) + return m != None + + # "^Using (\\d+) buckets" + def buckets(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Using (\d+) buckets", line) + if m: + entry.buckets = int(m.group(1)) + return m != None + + # Buffer size is: 4000MiB + def buffer_size(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r"^Buffer size is: (\d+)MiB", line) + if m: + entry.buffer = int(m.group(1)) + return m != None + + # Plot size is: 32 + def plot_size(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r'^Plot size is: (\d+)', line) + if m: + entry.plot_size = int(m.group(1)) + return m != None + + # Starting phase 1/4: Forward Propagation into tmp files... Sun May 9 17:36:12 2021 + def plot_start_date(self, line: str, entry: PlotInfo) -> bool: + m = re.search(r'^Starting phase 1/4: Forward Propagation into tmp files\.\.\. (.+)', line) + if m: + entry.started_at = m.group(1) + return m != None + + + # Ignore lines starting with Bucket + # Bucket 0 uniform sort. Ram: 3.250GiB, u_sort min: 0.563GiB, qs min: 0.281GiB. + def ignore_line(self, line: str, _: PlotInfo) -> bool: + return re.search(r'^\tBucket', line) \ No newline at end of file diff --git a/src/plotman/plotinfo.py b/src/plotman/plotinfo.py new file mode 100644 index 00000000..8b56c02c --- /dev/null +++ b/src/plotman/plotinfo.py @@ -0,0 +1,108 @@ +import attr + +@attr.s(repr=True, init=False) +class PlotInfo: + """Represents the results of a finished plot job""" + started_at: str = "" + plot_id: str = "" + buckets: int = 0 + threads: int = 0 + buffer: int = 0 + plot_size: int = 0 + tmp_dir1: str = "" + tmp_dir2: str = "" + phase1_duration_raw: float = 0 + phase2_duration_raw: float = 0 + phase3_duration_raw: float = 0 + phase4_duration_raw: float = 0 + total_time_raw: float = 0 + copy_time_raw: float = 0 + filename: str = "" + + def is_empty(self) -> bool: + "Data is considered empty if total_time is zero" + return self.total_time == 0 + + # Phase 1 duration + @property + def phase1_duration(self) -> int: + return round(self.phase1_duration_raw) + + @property + def phase1_duration_minutes(self) -> int: + return self.duration_to_minutes(self.phase1_duration_raw) + + @property + def phase1_duration_hours(self) -> float: + return self.duration_to_hours(self.phase1_duration_raw) + + # Phase 2 duration + @property + def phase2_duration(self) -> int: + return round(self.phase2_duration_raw) + + @property + def phase2_duration_minutes(self) -> int: + return self.duration_to_minutes(self.phase2_duration_raw) + + @property + def phase2_duration_hours(self) -> float: + return self.duration_to_hours(self.phase2_duration_raw) + + # Phase 3 duration + @property + def phase3_duration(self) -> int: + return round(self.phase3_duration_raw) + + @property + def phase3_duration_minutes(self) -> int: + return self.duration_to_minutes(self.phase3_duration_raw) + + @property + def phase3_duration_hours(self) -> float: + return self.duration_to_hours(self.phase3_duration_raw) + + # Phase 4 duration + @property + def phase4_duration(self) -> int: + return round(self.phase4_duration_raw) + + @property + def phase4_duration_minutes(self) -> int: + return self.duration_to_minutes(self.phase4_duration_raw) + + @property + def phase4_duration_hours(self) -> float: + return self.duration_to_hours(self.phase4_duration_raw) + + # Total time + @property + def total_time(self) -> int: + return round(self.total_time_raw) + + @property + def total_time_minutes(self) -> int: + return self.duration_to_minutes(self.total_time_raw) + + @property + def total_time_hours(self) -> float: + return self.duration_to_hours(self.total_time_raw) + + # Copy time + @property + def copy_time(self) -> int: + return round(self.copy_time_raw) + + @property + def copy_time_minutes(self) -> int: + return self.duration_to_minutes(self.copy_time_raw) + + @property + def copy_time_hours(self) -> float: + return self.duration_to_hours(self.copy_time_raw) + + def duration_to_minutes(self, duration): + return round(duration / 60) + + def duration_to_hours(self, duration): + return round(duration / 60 / 60, 2) From 04bbe883fd83a9d284eb753d3fc85729ef70d7b3 Mon Sep 17 00:00:00 2001 From: Rafael Steil Date: Wed, 19 May 2021 21:49:15 -0400 Subject: [PATCH 003/109] Add command to export log files to CSV --- src/plotman/csv_exporter.py | 115 ++++++++++++++++++++++++++++++++++++ src/plotman/plotman.py | 13 +++- 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 src/plotman/csv_exporter.py diff --git a/src/plotman/csv_exporter.py b/src/plotman/csv_exporter.py new file mode 100644 index 00000000..006c44ab --- /dev/null +++ b/src/plotman/csv_exporter.py @@ -0,0 +1,115 @@ +import csv +import sys +from dateutil.parser import parse as parse_date +from plotman.log_parser import PlotLogParser +from plotman.plotinfo import PlotInfo + +def export(logfilenames, save_to = None): + if save_to is None: + send_to_stdout(logfilenames) + else: + save_to_file(logfilenames, save_to) + +def save_to_file(logfilenames, filename: str): + with open(filename, 'w') as file: + generate(logfilenames, file) + +def send_to_stdout(logfilenames): + generate(logfilenames, sys.stdout) + +def header(writer): + writer.writerow([ + 'Plot ID', + 'Started at', + 'Date', + 'Size', + 'Buffer', + 'Buckets', + 'Threads', + 'Tmp dir 1', + 'Tmp dir 2', + 'Phase 1 duration (raw)', + 'Phase 1 duration', + 'Phase 1 duration (minutes)', + 'Phase 1 duration (hours)', + 'Phase 2 duration (raw)', + 'Phase 2 duration', + 'Phase 2 duration (minutes)', + 'Phase 2 duration (hours)', + 'Phase 3 duration (raw)', + 'Phase 3 duration', + 'Phase 3 duration (minutes)', + 'Phase 3 duration (hours)', + 'Phase 4 duration (raw)', + 'Phase 4 duration', + 'Phase 4 duration (minutes)', + 'Phase 4 duration (hours)', + 'Total time (raw)', + 'Total time', + 'Total time (minutes)', + 'Total time (hours)', + 'Copy time (raw)', + 'Copy time', + 'Copy time (minutes)', + 'Copy time (hours)', + 'Filename' + ]) + +def parse_logs(logfilenames): + parser = PlotLogParser() + result = [] + + for filename in logfilenames: + info = parser.parse(filename) + + if not info.is_empty(): + result.append(info) + + result.sort(key=log_sort_key) + return result + +def log_sort_key(element: PlotInfo): + return parse_date(element.started_at).replace(microsecond=0).isoformat() + +def generate(logfilenames, out): + writer = csv.writer(out) + header(writer) + logs = parse_logs(logfilenames) + + for info in logs: + writer.writerow([ + info.plot_id, + info.started_at, + parse_date(info.started_at).strftime('%Y-%m-%d'), + info.plot_size, + info.buffer, + info.buckets, + info.threads, + info.tmp_dir1, + info.tmp_dir2, + info.phase1_duration_raw, + info.phase1_duration, + info.phase1_duration_minutes, + info.phase1_duration_hours, + info.phase2_duration_raw, + info.phase2_duration, + info.phase2_duration_minutes, + info.phase2_duration_hours, + info.phase3_duration_raw, + info.phase3_duration, + info.phase3_duration_minutes, + info.phase3_duration_hours, + info.phase4_duration_raw, + info.phase4_duration, + info.phase4_duration_minutes, + info.phase4_duration_hours, + info.total_time_raw, + info.total_time, + info.total_time_minutes, + info.total_time_hours, + info.copy_time_raw, + info.copy_time, + info.copy_time_minutes, + info.copy_time_hours, + info.filename + ]) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 35e1616f..fa41d055 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -2,13 +2,14 @@ import importlib import importlib.resources import os +import glob import random from shutil import copyfile import time import datetime # Plotman libraries -from plotman import analyzer, archive, configuration, interactive, manager, plot_util, reporting +from plotman import analyzer, archive, configuration, interactive, manager, plot_util, reporting, csv_exporter from plotman import resources as plotman_resources from plotman.job import Job @@ -39,6 +40,9 @@ def parse_args(self): sp.add_parser('archive', help='move completed plots to farming location') + p_export = sp.add_parser('export', help='exports metadata from the plot logs as CSV') + p_export.add_argument('-o', dest='save_to', default=None, type=str, help='save to file. Optional, prints to stdout by default') + p_config = sp.add_parser('config', help='display or generate plotman.yaml configuration') sp_config = p_config.add_subparsers(dest='config_subcommand') sp_config.add_parser('generate', help='generate a default plotman.yaml file and print path') @@ -159,6 +163,13 @@ def main(): analyzer.analyze(args.logfile, args.clipterminals, args.bytmp, args.bybitfield) + # + # Exports log metadata to CSV + # + elif args.cmd == 'export': + logfilenames = glob.glob(os.path.join(cfg.directories.log, '*')) + csv_exporter.export(logfilenames, args.save_to) + else: jobs = Job.get_running_jobs(cfg.directories.log) From e2bc836bc4de00a67f3410e70b5470c1f92eb6fa Mon Sep 17 00:00:00 2001 From: djdjukic Date: Sun, 23 May 2021 21:17:09 +0200 Subject: [PATCH 004/109] Use tmp as dst when it is also a dst Modified manager.py to detect when a selected tmpdir is also configured as a dst, and, in that case, use it as the dstdir in order to avoid unneccessary copying. Useful for plotting directly to HDD. --- src/plotman/manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 8c418f35..0991ea19 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -96,11 +96,12 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg): # Plot to oldest tmpdir. tmpdir = max(rankable, key=operator.itemgetter(1))[0] - # Select the dst dir least recently selected dst_dir = dir_cfg.get_dst_directories() - if dir_cfg.dst_is_tmp(): + # Use tmpdir as dst if no dsts are configured, or if tmpdir is also configured as a dst + if dir_cfg.dst_is_tmp() or tmpdir in dst_dir: dstdir = tmpdir else: + # Select the dst dir least recently selected dir2ph = { d:ph for (d, ph) in dstdirs_to_youngest_phase(jobs).items() if d in dst_dir and ph is not None} unused_dirs = [d for d in dst_dir if d not in dir2ph.keys()] From 674e8511c90a2fa6801ab23d23d5d494e7f0c81f Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Tue, 8 Jun 2021 16:50:32 +0100 Subject: [PATCH 005/109] Update plotman.yaml Moved tmp_overrides into scheduling: to better reflect that they change the scheduling behaviour. Added more override options to allow different tmpdir characteristics to be catered for. --- src/plotman/resources/plotman.yaml | 33 ++++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 09ee52ad..21004605 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -44,19 +44,6 @@ directories: - /mnt/tmp/02 - /mnt/tmp/03 - # Optional: Allows overriding some characteristics of certain tmp - # directories. This contains a map of tmp directory names to - # attributes. If a tmp directory and attribute is not listed here, - # it uses the default attribute setting from the main configuration. - # - # Currently support override parameters: - # - tmpdir_max_jobs - tmp_overrides: - # In this example, /mnt/tmp/00 is larger than the other tmp - # dirs and it can hold more plots than the default. - "/mnt/tmp/00": - tmpdir_max_jobs: 5 - # Optional: tmp2 directory. If specified, will be passed to # chia plots create as -2. Only one tmp2 directory is supported. # tmp2: /mnt/tmp/a @@ -131,6 +118,26 @@ scheduling: # How often the daemon wakes to consider starting a new plot job, in seconds. polling_time_s: 20 + # Optional: Allows overriding some scheduling characteristics of certain tmp + # directories. This contains a map of tmp directory names to + # attributes. If a tmp directory and attribute is not listed here, + # it uses the default attribute setting from the main configuration. + # + # Currently support override parameters: + # - tmpdir_stagger_phase_major + # - tmpdir_stagger_phase_minor + # - tmpdir_stagger_phase_limit + # - tmpdir_max_jobs + tmp_overrides: + # In this example, /mnt/tmp/00 is larger than the other tmp + # dirs and it can hold more plots than the default, allowing + # more simultaneous plots and they are being started earlier + # than the global setting above. + "/mnt/tmp/00": + tmpdir_stagger_phase_major: 1 + tmpdir_stagger_phase_minr: 5 + tmpdir_stagger_phase_limit: 1 + tmpdir_max_jobs: 5 # Plotting parameters. These are pass-through parameters to chia plots create. # See documentation at From 15663757531f5154c4c484fbb3bfb72c00267ce3 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Tue, 8 Jun 2021 16:58:07 +0100 Subject: [PATCH 006/109] Update configuration.py Changes for reading the tmpdir overrides --- src/plotman/configuration.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 220dc538..06b82a78 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -186,6 +186,9 @@ def maybe_create_scripts(self, temp): @attr.frozen class TmpOverrides: + tmpdir_stagger_phase_major: Optional[int] = None + tmpdir_stagger_phase_minor: Optional[int] = None + tmpdir_stagger_phase_limit: Optional[int] = None tmpdir_max_jobs: Optional[int] = None @attr.frozen @@ -222,7 +225,6 @@ class Directories: tmp: List[str] dst: Optional[List[str]] = None tmp2: Optional[str] = None - tmp_overrides: Optional[Dict[str, TmpOverrides]] = None def dst_is_tmp(self): return self.dst is None and self.tmp2 is None @@ -250,6 +252,7 @@ class Scheduling: tmpdir_stagger_phase_major: int tmpdir_stagger_phase_minor: int tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 + tmp_overrides: Optional[Dict[str, TmpOverrides]] = None @attr.frozen class Plotting: From f253d060d6531c754a4796e9b625ee94b733f331 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Tue, 8 Jun 2021 17:12:47 +0100 Subject: [PATCH 007/109] Update manager.py --- src/plotman/manager.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 7c5b116d..4982f2a9 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -53,21 +53,34 @@ def phases_permit_new_job(phases, d, sched_cfg, dir_cfg): if len(phases) == 0: return True - milestone = job.Phase( - major=sched_cfg.tmpdir_stagger_phase_major, - minor=sched_cfg.tmpdir_stagger_phase_minor, - ) + # Check if any overrides exist for the current job + if sched_cfg.tmp_overrides is not None and d in sched_cfg.tmp_overrides: + curr_overrides = sched_cfg.tmp_overrides[d] + + # Check and apply overrides for major, minor and phase limit + if curr_overrides.tmpdir_stagger_phase_major is not None: + major = curr_overrides.tmpdir_stagger_phase_major + else: + major = sched_cfg.tmpdir_stagger_phase_major + if curr_overrides.tmpdir_stagger_phase_minor is not None: + minor = curr_overrides.tmpdir_stagger_phase_minor + else: + minor = sched_cfg.tmpdir_stagger_phase_minor + milestone = job.Phase(major,minor,) + if curr_overrides.tmpdir_stagger_phase_limit is not None: + stagger_phase_limit = curr_overrides.tmpdir_stagger_phase_limit + else: + stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py - if len([p for p in phases if p < milestone]) >= sched_cfg.tmpdir_stagger_phase_limit: + if len([p for p in phases if p < milestone]) >= stagger_phase_limit: return False # Limit the total number of jobs per tmp dir. Default to the overall max # jobs configuration, but restrict to any configured overrides. - max_plots = sched_cfg.tmpdir_max_jobs - if dir_cfg.tmp_overrides is not None and d in dir_cfg.tmp_overrides: - curr_overrides = dir_cfg.tmp_overrides[d] - if curr_overrides.tmpdir_max_jobs is not None: - max_plots = curr_overrides.tmpdir_max_jobs + if curr_overrides.tmpdir_max_jobs is not None: + max_plots = curr_overrides.tmpdir_max_jobs + else: + max_plots = sched_cfg.tmpdir_max_jobs if len(phases) >= max_plots: return False From 082c1e4deebb94daf2d6580bd30d1db5b9792e66 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Tue, 8 Jun 2021 17:59:18 +0100 Subject: [PATCH 008/109] Update plotman.yaml --- src/plotman/resources/plotman.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 21004605..9835d16e 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -135,7 +135,7 @@ scheduling: # than the global setting above. "/mnt/tmp/00": tmpdir_stagger_phase_major: 1 - tmpdir_stagger_phase_minr: 5 + tmpdir_stagger_phase_minor: 5 tmpdir_stagger_phase_limit: 1 tmpdir_max_jobs: 5 From 9d0bc0deca366b70e78677e932b8b4a2b7aacc5a Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 11 Jun 2021 00:16:09 -0400 Subject: [PATCH 009/109] v0.4+dev --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bd73f470..fa939db4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4 +0.4+dev From 9b1b829b4878e6cac99608929f0f14f6295461ff Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 11 Jun 2021 19:58:59 -0400 Subject: [PATCH 010/109] tweaks --- src/plotman/_tests/log_parser_test.py | 101 ++++++++++++++------------ src/plotman/log_parser.py | 16 ++-- src/plotman/plotinfo.py | 12 ++- 3 files changed, 70 insertions(+), 59 deletions(-) diff --git a/src/plotman/_tests/log_parser_test.py b/src/plotman/_tests/log_parser_test.py index 31e6c417..4a8a1b16 100644 --- a/src/plotman/_tests/log_parser_test.py +++ b/src/plotman/_tests/log_parser_test.py @@ -2,53 +2,58 @@ from plotman._tests import resources from plotman.log_parser import PlotLogParser +import plotman.job +import plotman.plotinfo + +example_info = plotman.plotinfo.PlotInfo( + started_at=plotman.job.parse_chia_plot_time(s="Sun Apr 4 19:00:50 2021"), + plot_id="3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24", + buckets=128, + threads=4, + buffer=4000, + plot_size=32, + tmp_dir1="/farm/yards/901", + tmp_dir2="/farm/yards/901", + phase1_duration_raw=17571.981, + phase2_duration_raw=6911.621, + phase3_duration_raw=14537.188, + phase4_duration_raw=924.288, + total_time_raw=39945.080, + copy_time_raw=501.696, + filename="/farm/wagons/801/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot", +) + def test_should_correctly_parse(): - with importlib.resources.path(resources, "2021-04-04T19_00_47.681088-0400.log") as p: - logfile_path = p - - parser = PlotLogParser() - info = parser.parse(logfile_path) - - assert info.plot_id == "3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24" - - assert info.plot_size == 32 - assert info.started_at == "Sun Apr 4 19:00:50 2021" - assert info.buffer == 4000 - assert info.buckets == 128 - assert info.threads == 4 - - assert info.tmp_dir1 == "/farm/yards/901" - assert info.tmp_dir2 == "/farm/yards/901" - - assert info.phase1_duration_raw == 17571.981 - assert info.phase1_duration == 17572 - assert info.phase1_duration_minutes == 293 - assert info.phase1_duration_hours == 4.88 - - assert info.phase2_duration_raw == 6911.621 - assert info.phase2_duration == 6912 - assert info.phase2_duration_minutes == 115 - assert info.phase2_duration_hours == 1.92 - - assert info.phase3_duration_raw == 14537.188 - assert info.phase3_duration == 14537 - assert info.phase3_duration_minutes == 242 - assert info.phase3_duration_hours == 4.04 - - assert info.phase4_duration_raw == 924.288 - assert info.phase4_duration == 924 - assert info.phase4_duration_minutes == 15 - assert info.phase4_duration_hours == 0.26 - - assert info.total_time_raw == 39945.080 - assert info.total_time == 39945 - assert info.total_time_minutes == 666 - assert info.total_time_hours == 11.10 - - assert info.copy_time_raw == 501.696 - assert info.copy_time == 502 - assert info.copy_time_minutes == 8 - assert info.copy_time_hours == 0.14 - - assert info.filename == "/farm/wagons/801/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot" \ No newline at end of file + with importlib.resources.open_text( + resources, + "2021-04-04T19_00_47.681088-0400.log", + ) as file: + parser = PlotLogParser() + info = parser.parse(file) + + assert info == example_info + + assert info.phase1_duration == 17572 + assert info.phase1_duration_minutes == 293 + assert info.phase1_duration_hours == 4.88 + + assert info.phase2_duration == 6912 + assert info.phase2_duration_minutes == 115 + assert info.phase2_duration_hours == 1.92 + + assert info.phase3_duration == 14537 + assert info.phase3_duration_minutes == 242 + assert info.phase3_duration_hours == 4.04 + + assert info.phase4_duration == 924 + assert info.phase4_duration_minutes == 15 + assert info.phase4_duration_hours == 0.26 + + assert info.total_time == 39945 + assert info.total_time_minutes == 666 + assert info.total_time_hours == 11.10 + + assert info.copy_time == 502 + assert info.copy_time_minutes == 8 + assert info.copy_time_hours == 0.14 diff --git a/src/plotman/log_parser.py b/src/plotman/log_parser.py index 5dbfcfe0..c125c584 100644 --- a/src/plotman/log_parser.py +++ b/src/plotman/log_parser.py @@ -1,10 +1,13 @@ +import os import re from plotman.plotinfo import PlotInfo +import plotman.job + class PlotLogParser: """Parser for a finished plotting job""" - def parse(self, filename: str) -> PlotInfo: + def parse(self, file) -> PlotInfo: """Parses a single log and returns its info""" entry = PlotInfo() @@ -26,11 +29,10 @@ def parse(self, filename: str) -> PlotInfo: self.filename ] - with open(filename, 'r') as f: - for line in f: - for matcher in matchers: - if (matcher(line, entry)): - break + for line in file: + for matcher in matchers: + if (matcher(line, entry)): + break return entry @@ -130,7 +132,7 @@ def plot_size(self, line: str, entry: PlotInfo) -> bool: def plot_start_date(self, line: str, entry: PlotInfo) -> bool: m = re.search(r'^Starting phase 1/4: Forward Propagation into tmp files\.\.\. (.+)', line) if m: - entry.started_at = m.group(1) + entry.started_at = plotman.job.parse_chia_plot_time(s=m.group(1)) return m != None diff --git a/src/plotman/plotinfo.py b/src/plotman/plotinfo.py index 8b56c02c..0f131c16 100644 --- a/src/plotman/plotinfo.py +++ b/src/plotman/plotinfo.py @@ -1,9 +1,13 @@ +import typing + import attr +import pendulum + -@attr.s(repr=True, init=False) +@attr.mutable class PlotInfo: """Represents the results of a finished plot job""" - started_at: str = "" + started_at: typing.Optional[pendulum.DateTime] = None plot_id: str = "" buckets: int = 0 threads: int = 0 @@ -19,8 +23,8 @@ class PlotInfo: copy_time_raw: float = 0 filename: str = "" - def is_empty(self) -> bool: - "Data is considered empty if total_time is zero" + def in_progress(self) -> bool: + "The plot is in progress if no total time has been reported." return self.total_time == 0 # Phase 1 duration From f65809346d3a5a5e06469cc90e80d187dc4aaca3 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 11 Jun 2021 21:29:59 -0400 Subject: [PATCH 011/109] tweak --- src/plotman/csv_exporter.py | 190 +++++++++++++++++++----------------- src/plotman/plotman.py | 21 ++-- 2 files changed, 111 insertions(+), 100 deletions(-) diff --git a/src/plotman/csv_exporter.py b/src/plotman/csv_exporter.py index 006c44ab..0a1b5ade 100644 --- a/src/plotman/csv_exporter.py +++ b/src/plotman/csv_exporter.py @@ -1,115 +1,121 @@ import csv import sys from dateutil.parser import parse as parse_date + +import attr + from plotman.log_parser import PlotLogParser -from plotman.plotinfo import PlotInfo -def export(logfilenames, save_to = None): - if save_to is None: - send_to_stdout(logfilenames) - else: - save_to_file(logfilenames, save_to) +def row_ib(name): + return attr.ib(converter=str, metadata={'name': name}) -def save_to_file(logfilenames, filename: str): - with open(filename, 'w') as file: - generate(logfilenames, file) +@attr.frozen +class Row: + plot_id: str = row_ib(name='Plot ID') + started_at: str = row_ib(name='Started at') + date: str = row_ib(name='Date') + size: str = row_ib(name='Size') + buffer: str = row_ib(name='Buffer') + buckets: str = row_ib(name='Buckets') + threads: str = row_ib(name='Threads') + tmp_dir_1: str = row_ib(name='Tmp dir 1') + tmp_dir_2: str = row_ib(name='Tmp dir 2') + phase_1_duration_raw: str = row_ib(name='Phase 1 duration (raw)') + phase_1_duration: str = row_ib(name='Phase 1 duration') + phase_1_duration_minutes: str = row_ib(name='Phase 1 duration (minutes)') + phase_1_duration_hours: str = row_ib(name='Phase 1 duration (hours)') + phase_2_duration_raw: str = row_ib(name='Phase 2 duration (raw)') + phase_2_duration: str = row_ib(name='Phase 2 duration') + phase_2_duration_minutes: str = row_ib(name='Phase 2 duration (minutes)') + phase_2_duration_hours: str = row_ib(name='Phase 2 duration (hours)') + phase_3_duration_raw: str = row_ib(name='Phase 3 duration (raw)') + phase_3_duration: str = row_ib(name='Phase 3 duration') + phase_3_duration_minutes: str = row_ib(name='Phase 3 duration (minutes)') + phase_3_duration_hours: str = row_ib(name='Phase 3 duration (hours)') + phase_4_duration_raw: str = row_ib(name='Phase 4 duration (raw)') + phase_4_duration: str = row_ib(name='Phase 4 duration') + phase_4_duration_minutes: str = row_ib(name='Phase 4 duration (minutes)') + phase_4_duration_hours: str = row_ib(name='Phase 4 duration (hours)') + total_time_raw: str = row_ib(name='Total time (raw)') + total_time: str = row_ib(name='Total time') + total_time_minutes: str = row_ib(name='Total time (minutes)') + total_time_hours: str = row_ib(name='Total time (hours)') + copy_time_raw: str = row_ib(name='Copy time (raw)') + copy_time: str = row_ib(name='Copy time') + copy_time_minutes: str = row_ib(name='Copy time (minutes)') + copy_time_hours: str = row_ib(name='Copy time (hours)') + filename: str = row_ib(name='Filename') -def send_to_stdout(logfilenames): - generate(logfilenames, sys.stdout) + @classmethod + def names(cls): + return [field.metadata['name'] for field in attr.fields(cls)] -def header(writer): - writer.writerow([ - 'Plot ID', - 'Started at', - 'Date', - 'Size', - 'Buffer', - 'Buckets', - 'Threads', - 'Tmp dir 1', - 'Tmp dir 2', - 'Phase 1 duration (raw)', - 'Phase 1 duration', - 'Phase 1 duration (minutes)', - 'Phase 1 duration (hours)', - 'Phase 2 duration (raw)', - 'Phase 2 duration', - 'Phase 2 duration (minutes)', - 'Phase 2 duration (hours)', - 'Phase 3 duration (raw)', - 'Phase 3 duration', - 'Phase 3 duration (minutes)', - 'Phase 3 duration (hours)', - 'Phase 4 duration (raw)', - 'Phase 4 duration', - 'Phase 4 duration (minutes)', - 'Phase 4 duration (hours)', - 'Total time (raw)', - 'Total time', - 'Total time (minutes)', - 'Total time (hours)', - 'Copy time (raw)', - 'Copy time', - 'Copy time (minutes)', - 'Copy time (hours)', - 'Filename' - ]) + @classmethod + def from_info(cls, info): + return cls( + plot_id=info.plot_id, + started_at=info.started_at.isoformat(), + date=info.started_at.date().isoformat(), + size=info.plot_size, + buffer=info.buffer, + buckets=info.buckets, + threads=info.threads, + tmp_dir_1=info.tmp_dir1, + tmp_dir_2=info.tmp_dir2, + phase_1_duration_raw=info.phase1_duration_raw, + phase_1_duration=info.phase1_duration, + phase_1_duration_minutes=info.phase1_duration_minutes, + phase_1_duration_hours=info.phase1_duration_hours, + phase_2_duration_raw=info.phase2_duration_raw, + phase_2_duration=info.phase2_duration, + phase_2_duration_minutes=info.phase2_duration_minutes, + phase_2_duration_hours=info.phase2_duration_hours, + phase_3_duration_raw=info.phase3_duration_raw, + phase_3_duration=info.phase3_duration, + phase_3_duration_minutes=info.phase3_duration_minutes, + phase_3_duration_hours=info.phase3_duration_hours, + phase_4_duration_raw=info.phase4_duration_raw, + phase_4_duration=info.phase4_duration, + phase_4_duration_minutes=info.phase4_duration_minutes, + phase_4_duration_hours=info.phase4_duration_hours, + total_time_raw=info.total_time_raw, + total_time=info.total_time, + total_time_minutes=info.total_time_minutes, + total_time_hours=info.total_time_hours, + copy_time_raw=info.copy_time_raw, + copy_time=info.copy_time, + copy_time_minutes=info.copy_time_minutes, + copy_time_hours=info.copy_time_hours, + filename=info.filename, + ) + + def name_dict(self): + return { + field.metadata['name']: value + for field, value in zip(attr.fields(type(self)), attr.astuple(self)) + } def parse_logs(logfilenames): parser = PlotLogParser() result = [] for filename in logfilenames: - info = parser.parse(filename) + with open(filename) as file: + info = parser.parse(file) - if not info.is_empty(): + if not info.in_progress(): result.append(info) - result.sort(key=log_sort_key) + result.sort(key=lambda element: element.started_at) return result -def log_sort_key(element: PlotInfo): - return parse_date(element.started_at).replace(microsecond=0).isoformat() -def generate(logfilenames, out): - writer = csv.writer(out) - header(writer) +def generate(logfilenames, file): + writer = csv.DictWriter(file, fieldnames=Row.names()) + writer.writeheader() + logs = parse_logs(logfilenames) for info in logs: - writer.writerow([ - info.plot_id, - info.started_at, - parse_date(info.started_at).strftime('%Y-%m-%d'), - info.plot_size, - info.buffer, - info.buckets, - info.threads, - info.tmp_dir1, - info.tmp_dir2, - info.phase1_duration_raw, - info.phase1_duration, - info.phase1_duration_minutes, - info.phase1_duration_hours, - info.phase2_duration_raw, - info.phase2_duration, - info.phase2_duration_minutes, - info.phase2_duration_hours, - info.phase3_duration_raw, - info.phase3_duration, - info.phase3_duration_minutes, - info.phase3_duration_hours, - info.phase4_duration_raw, - info.phase4_duration, - info.phase4_duration_minutes, - info.phase4_duration_hours, - info.total_time_raw, - info.total_time, - info.total_time_minutes, - info.total_time_hours, - info.copy_time_raw, - info.copy_time, - info.copy_time_minutes, - info.copy_time_hours, - info.filename - ]) + row = Row.from_info(info=info) + writer.writerow(rowdict=row.name_dict()) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 4f79c85c..02f4090c 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -8,6 +8,7 @@ import glob import random from shutil import copyfile +import sys import time import pendulum @@ -193,14 +194,18 @@ def main(): args.bytmp, args.bybitfield) # - # Exports log metadata to CSV - # - elif args.cmd == 'export': - logfilenames = glob.glob(os.path.join(cfg.directories.log, '*')) - csv_exporter.export(logfilenames, args.save_to) - - else: - jobs = Job.get_running_jobs(cfg.logging.plots) + # Exports log metadata to CSV + # + elif args.cmd == 'export': + logfilenames = glob.glob(os.path.join(cfg.logging.plots, '*.plot.log')) + if args.save_to is None: + csv_exporter.generate(logfilenames=logfilenames, file=sys.stdout) + else: + with open(args.save_to, 'w', encoding='utf-8') as file: + csv_exporter.generate(logfilenames=logfilenames, file=file) + + else: + jobs = Job.get_running_jobs(cfg.logging.plots) # Status report if args.cmd == 'status': From 326aedda15be0f0df926c5ecc4a92e9b2545be29 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 12 Jun 2021 20:41:18 -0400 Subject: [PATCH 012/109] mypy --- .github/workflows/ci.yml | 6 +- mypy.ini | 25 +++++ setup.cfg | 5 +- src/plotman/_tests/archive_test.py | 2 +- src/plotman/_tests/configuration_test.py | 22 ++-- src/plotman/_tests/job_test.py | 30 +++--- src/plotman/_tests/log_parser_test.py | 2 +- src/plotman/_tests/manager_test.py | 57 +++++----- src/plotman/_tests/plot_util_test.py | 12 +-- src/plotman/_tests/reporting_test.py | 25 ++--- src/plotman/analyzer.py | 7 +- src/plotman/archive.py | 30 +++--- src/plotman/chia.py | 36 ++++--- src/plotman/chiapos.py | 10 +- src/plotman/configuration.py | 72 +++++++------ src/plotman/csv_exporter.py | 101 +++++++++-------- src/plotman/interactive.py | 41 +++---- src/plotman/job.py | 131 ++++++++++++++--------- src/plotman/log_parser.py | 7 +- src/plotman/manager.py | 38 ++++--- src/plotman/plot_util.py | 56 +++++----- src/plotman/plotinfo.py | 4 +- src/plotman/plotman.py | 50 +++++---- src/plotman/reporting.py | 38 +++---- tox.ini | 10 +- 25 files changed, 479 insertions(+), 338 deletions(-) create mode 100644 mypy.ini diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2c0f406..bd920689 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,10 +107,12 @@ jobs: - name: Test tox: test coverage: true + - name: Check hints + tox: check-hints include: - task: - name: Check - tox: check + name: Check manifest + tox: check-manifest os: name: Linux runs-on: ubuntu-latest diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..71319f45 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,25 @@ +[mypy] +show_error_codes = true +strict = true + +[mypy-appdirs] +ignore_missing_imports = true + +[mypy-click] +ignore_missing_imports = true + +[mypy-pendulum] +# TODO: https://github.com/sdispater/pendulum/pull/551 +implicit_reexport = true + +[mypy-psutil] +ignore_missing_imports = true + +[mypy-pyfakefs] +ignore_missing_imports = true + +[mypy-texttable] +ignore_missing_imports = true + +[mypy-yaml] +ignore_missing_imports = true diff --git a/setup.cfg b/setup.cfg index a85356b9..551397ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,10 +62,13 @@ dev = isort test = %(coverage)s - check-manifest pytest pytest-cov pyfakefs +checks = + check-manifest ~= 0.46 + mypy == 0.902 + types-pkg_resources ~= 0.1.2 [options.data_files] config = src/plotman/resources/plotman.yaml diff --git a/src/plotman/_tests/archive_test.py b/src/plotman/_tests/archive_test.py index 62ec1570..ddd05e75 100755 --- a/src/plotman/_tests/archive_test.py +++ b/src/plotman/_tests/archive_test.py @@ -1,6 +1,6 @@ from plotman import archive, job -def test_compute_priority(): +def test_compute_priority() -> None: assert (archive.compute_priority( job.Phase(major=3, minor=1), 1000, 10) > archive.compute_priority( job.Phase(major=3, minor=6), 1000, 10) ) diff --git a/src/plotman/_tests/configuration_test.py b/src/plotman/_tests/configuration_test.py index e777f08f..db4aa41e 100644 --- a/src/plotman/_tests/configuration_test.py +++ b/src/plotman/_tests/configuration_test.py @@ -9,23 +9,23 @@ @pytest.fixture(name='config_text') -def config_text_fixture(): +def config_text_fixture() -> str: return importlib.resources.read_text(plotman_resources, "plotman.yaml") @pytest.fixture(name='target_definitions_text') -def target_definitions_text_fixture(): +def target_definitions_text_fixture() -> str: return importlib.resources.read_text( plotman_resources, "target_definitions.yaml", ) -def test_get_validated_configs__default(config_text, target_definitions_text): +def test_get_validated_configs__default(config_text: str, target_definitions_text: str) -> None: """Check that get_validated_configs() works with default/example plotman.yaml file.""" res = configuration.get_validated_configs(config_text, '', target_definitions_text) assert isinstance(res, configuration.PlotmanConfig) -def test_get_validated_configs__malformed(config_text, target_definitions_text): +def test_get_validated_configs__malformed(config_text: str, target_definitions_text: str) -> None: """Check that get_validated_configs() raises exception with invalid plotman.yaml contents.""" loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader) @@ -39,7 +39,7 @@ def test_get_validated_configs__malformed(config_text, target_definitions_text): assert exc_info.value.args[0] == f"Config file at: '/the_path' is malformed" -def test_get_validated_configs__missing(): +def test_get_validated_configs__missing() -> None: """Check that get_validated_configs() raises exception when plotman.yaml does not exist.""" with pytest.raises(configuration.ConfigurationException) as exc_info: configuration.read_configuration_text('/invalid_path') @@ -50,7 +50,7 @@ def test_get_validated_configs__missing(): ) -def test_loads_without_user_interface(config_text, target_definitions_text): +def test_loads_without_user_interface(config_text: str, target_definitions_text: str) -> None: loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader) del loaded_yaml["user_interface"] @@ -62,7 +62,7 @@ def test_loads_without_user_interface(config_text, target_definitions_text): assert reloaded_yaml.user_interface == configuration.UserInterface() -def test_loads_without_user_archiving(config_text, target_definitions_text): +def test_loads_without_user_archiving(config_text: str, target_definitions_text: str) -> None: loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader) del loaded_yaml["archiving"] @@ -74,7 +74,7 @@ def test_loads_without_user_archiving(config_text, target_definitions_text): assert reloaded_yaml.archiving is None -def test_get_dst_directories_gets_dst(): +def test_get_dst_directories_gets_dst() -> None: tmp = ['/tmp'] dst = ['/dst0', '/dst1'] directories = configuration.Directories(tmp=tmp, dst=dst) @@ -82,14 +82,14 @@ def test_get_dst_directories_gets_dst(): assert directories.get_dst_directories() == dst -def test_get_dst_directories_gets_tmp(): +def test_get_dst_directories_gets_tmp() -> None: tmp = ['/tmp'] directories = configuration.Directories(tmp=tmp) assert directories.get_dst_directories() == tmp -def test_dst_is_dst(): +def test_dst_is_dst() -> None: tmp = ['/tmp'] dst = ['/dst0', '/dst1'] directories = configuration.Directories(tmp=tmp, dst=dst) @@ -97,7 +97,7 @@ def test_dst_is_dst(): assert not directories.dst_is_tmp() -def test_dst_is_tmp(): +def test_dst_is_tmp() -> None: tmp = ['/tmp'] directories = configuration.Directories(tmp=tmp) diff --git a/src/plotman/_tests/job_test.py b/src/plotman/_tests/job_test.py index 25840723..84269f40 100644 --- a/src/plotman/_tests/job_test.py +++ b/src/plotman/_tests/job_test.py @@ -2,7 +2,11 @@ import datetime import locale import importlib.resources +import os +import pathlib +import typing +import pendulum import pytest from plotman import job from plotman._tests import resources @@ -12,15 +16,17 @@ class FauxJobWithLogfile: # plotman.job.Job does too much in its .__init_() so we have this to let us # test its .init_from_logfile(). - def __init__(self, logfile_path): + start_time: pendulum.DateTime + + def __init__(self, logfile_path: str) -> None: self.logfile = logfile_path - def update_from_logfile(self): + def update_from_logfile(self) -> None: pass @pytest.fixture(name='logfile_path') -def logfile_fixture(tmp_path): +def logfile_fixture(tmp_path: pathlib.Path) -> pathlib.Path: log_name = '2021-04-04T19_00_47.681088-0400.log' log_contents = importlib.resources.read_binary(resources, log_name) log_file_path = tmp_path.joinpath(log_name) @@ -30,7 +36,7 @@ def logfile_fixture(tmp_path): @contextlib.contextmanager -def set_locale(name): +def set_locale(name: str) -> typing.Generator[str, None, None]: # This is terrible and not thread safe. original = locale.setlocale(locale.LC_ALL) @@ -47,11 +53,11 @@ def set_locale(name): argnames=['locale_name'], argvalues=[['C'], ['en_US.UTF-8'], ['de_DE.UTF-8']], ) -def test_job_parses_time_with_non_english_locale(logfile_path, locale_name): - faux_job_with_logfile = FauxJobWithLogfile(logfile_path=logfile_path) +def test_job_parses_time_with_non_english_locale(logfile_path: pathlib.Path, locale_name: str) -> None: + faux_job_with_logfile = FauxJobWithLogfile(logfile_path=os.fspath(logfile_path)) with set_locale(locale_name): - job.Job.init_from_logfile(self=faux_job_with_logfile) + job.Job.init_from_logfile(self=faux_job_with_logfile) # type: ignore[arg-type] assert faux_job_with_logfile.start_time == log_file_time @@ -67,7 +73,7 @@ def test_job_parses_time_with_non_english_locale(logfile_path, locale_name): ], ids=str, ) -def test_chia_plots_create_parsing_does_not_fail(arguments): +def test_chia_plots_create_parsing_does_not_fail(arguments: typing.List[str]) -> None: job.parse_chia_plots_create_command_line( command_line=['python', 'chia', 'plots', 'create', *arguments], ) @@ -82,7 +88,7 @@ def test_chia_plots_create_parsing_does_not_fail(arguments): ], ids=str, ) -def test_chia_plots_create_parsing_detects_help(arguments): +def test_chia_plots_create_parsing_detects_help(arguments: typing.List[str]) -> None: parsed = job.parse_chia_plots_create_command_line( command_line=['python', 'chia', 'plots', 'create', *arguments], ) @@ -99,7 +105,7 @@ def test_chia_plots_create_parsing_detects_help(arguments): ], ids=str, ) -def test_chia_plots_create_parsing_detects_not_help(arguments): +def test_chia_plots_create_parsing_detects_not_help(arguments: typing.List[str]) -> None: parsed = job.parse_chia_plots_create_command_line( command_line=['python', 'chia', 'plots', 'create', *arguments], ) @@ -117,7 +123,7 @@ def test_chia_plots_create_parsing_detects_not_help(arguments): ], ids=str, ) -def test_chia_plots_create_parsing_handles_argument_forms(arguments): +def test_chia_plots_create_parsing_handles_argument_forms(arguments: typing.List[str]) -> None: parsed = job.parse_chia_plots_create_command_line( command_line=['python', 'chia', 'plots', 'create', *arguments], ) @@ -133,7 +139,7 @@ def test_chia_plots_create_parsing_handles_argument_forms(arguments): ], ids=str, ) -def test_chia_plots_create_parsing_identifies_errors(arguments): +def test_chia_plots_create_parsing_identifies_errors(arguments: typing.List[str]) -> None: parsed = job.parse_chia_plots_create_command_line( command_line=['python', 'chia', 'plots', 'create', *arguments], ) diff --git a/src/plotman/_tests/log_parser_test.py b/src/plotman/_tests/log_parser_test.py index 4a8a1b16..4d9f6db7 100644 --- a/src/plotman/_tests/log_parser_test.py +++ b/src/plotman/_tests/log_parser_test.py @@ -24,7 +24,7 @@ ) -def test_should_correctly_parse(): +def test_should_correctly_parse() -> None: with importlib.resources.open_text( resources, "2021-04-04T19_00_47.681088-0400.log", diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py index 7b1d2573..0c0b752e 100755 --- a/src/plotman/_tests/manager_test.py +++ b/src/plotman/_tests/manager_test.py @@ -1,3 +1,4 @@ +import typing # TODO: migrate away from unittest patch from unittest.mock import patch @@ -7,7 +8,7 @@ @pytest.fixture -def sched_cfg(): +def sched_cfg() -> configuration.Scheduling: return configuration.Scheduling( global_max_jobs=1, global_stagger_m=2, @@ -18,39 +19,39 @@ def sched_cfg(): ) @pytest.fixture -def dir_cfg(): +def dir_cfg() -> configuration.Directories: return configuration.Directories( tmp=["/var/tmp", "/tmp"], dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"], tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)} ) -def test_permit_new_job_post_milestone(sched_cfg, dir_cfg): +def test_permit_new_job_post_milestone(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None: phases = job.Phase.list_from_tuples([ (3, 8), (4, 1) ]) assert manager.phases_permit_new_job( phases, '/mnt/tmp/00', sched_cfg, dir_cfg) -def test_permit_new_job_pre_milestone(sched_cfg, dir_cfg): +def test_permit_new_job_pre_milestone(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None: phases = job.Phase.list_from_tuples([ (2, 3), (4, 1) ]) assert not manager.phases_permit_new_job( phases, '/mnt/tmp/00', sched_cfg, dir_cfg) -def test_permit_new_job_too_many_jobs(sched_cfg, dir_cfg): +def test_permit_new_job_too_many_jobs(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None: phases = job.Phase.list_from_tuples([ (3, 1), (3, 2), (3, 3) ]) assert not manager.phases_permit_new_job( phases, '/mnt/tmp/00', sched_cfg, dir_cfg) -def test_permit_new_job_too_many_jobs_zerophase(sched_cfg, dir_cfg): +def test_permit_new_job_too_many_jobs_zerophase(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None: phases = job.Phase.list_from_tuples([ (3, 0), (3, 1), (3, 3) ]) assert not manager.phases_permit_new_job( phases, '/mnt/tmp/00', sched_cfg, dir_cfg) -def test_permit_new_job_too_many_jobs_nonephase(sched_cfg, dir_cfg): +def test_permit_new_job_too_many_jobs_nonephase(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None: phases = job.Phase.list_from_tuples([ (None, None), (3, 1), (3, 3) ]) assert manager.phases_permit_new_job( phases, '/mnt/tmp/00', sched_cfg, dir_cfg) -def test_permit_new_job_override_tmp_dir(sched_cfg, dir_cfg): +def test_permit_new_job_override_tmp_dir(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None: phases = job.Phase.list_from_tuples([ (3, 1), (3, 2), (3, 3) ]) assert manager.phases_permit_new_job( phases, '/mnt/tmp/04', sched_cfg, dir_cfg) @@ -60,40 +61,40 @@ def test_permit_new_job_override_tmp_dir(sched_cfg, dir_cfg): dir_cfg) @patch('plotman.job.Job') -def job_w_tmpdir_phase(tmpdir, phase, MockJob): +def job_w_tmpdir_phase(tmpdir: str, phase: job.Phase, MockJob: typing.Any) -> typing.Any: j = MockJob() j.progress.return_value = phase j.tmpdir = tmpdir return j @patch('plotman.job.Job') -def job_w_dstdir_phase(dstdir, phase, MockJob): +def job_w_dstdir_phase(dstdir: str, phase: job.Phase, MockJob: typing.Any) -> typing.Any: j = MockJob() j.progress.return_value = phase j.dstdir = dstdir return j -def test_dstdirs_to_furthest_phase(): - all_jobs = [ job_w_dstdir_phase('/plots1', (1, 5)), - job_w_dstdir_phase('/plots2', (1, 1)), - job_w_dstdir_phase('/plots2', (3, 1)), - job_w_dstdir_phase('/plots2', (2, 1)), - job_w_dstdir_phase('/plots3', (4, 1)) ] +def test_dstdirs_to_furthest_phase() -> None: + all_jobs = [ job_w_dstdir_phase('/plots1', job.Phase(1, 5)), + job_w_dstdir_phase('/plots2', job.Phase(1, 1)), + job_w_dstdir_phase('/plots2', job.Phase(3, 1)), + job_w_dstdir_phase('/plots2', job.Phase(2, 1)), + job_w_dstdir_phase('/plots3', job.Phase(4, 1)) ] assert (manager.dstdirs_to_furthest_phase(all_jobs) == - { '/plots1' : (1, 5), - '/plots2' : (3, 1), - '/plots3' : (4, 1) } ) + { '/plots1' : job.Phase(1, 5), + '/plots2' : job.Phase(3, 1), + '/plots3' : job.Phase(4, 1) } ) -def test_dstdirs_to_youngest_phase(): - all_jobs = [ job_w_dstdir_phase('/plots1', (1, 5)), - job_w_dstdir_phase('/plots2', (1, 1)), - job_w_dstdir_phase('/plots2', (3, 1)), - job_w_dstdir_phase('/plots2', (2, 1)), - job_w_dstdir_phase('/plots3', (4, 1)) ] +def test_dstdirs_to_youngest_phase() -> None: + all_jobs = [ job_w_dstdir_phase('/plots1', job.Phase(1, 5)), + job_w_dstdir_phase('/plots2', job.Phase(1, 1)), + job_w_dstdir_phase('/plots2', job.Phase(3, 1)), + job_w_dstdir_phase('/plots2', job.Phase(2, 1)), + job_w_dstdir_phase('/plots3', job.Phase(4, 1)) ] assert (manager.dstdirs_to_youngest_phase(all_jobs) == - { '/plots1' : (1, 5), - '/plots2' : (1, 1), - '/plots3' : (4, 1) } ) + { '/plots1' : job.Phase(1, 5), + '/plots2' : job.Phase(1, 1), + '/plots3' : job.Phase(4, 1) } ) diff --git a/src/plotman/_tests/plot_util_test.py b/src/plotman/_tests/plot_util_test.py index d0f8aef2..4a9a8c0b 100755 --- a/src/plotman/_tests/plot_util_test.py +++ b/src/plotman/_tests/plot_util_test.py @@ -6,7 +6,7 @@ from plotman.plot_util import GB -def test_human_format(): +def test_human_format() -> None: assert (plot_util.human_format(3442000000, 0) == '3G') assert (plot_util.human_format(3542000, 2) == '3.54M') assert (plot_util.human_format(354, 0) == '354') @@ -15,7 +15,7 @@ def test_human_format(): assert (plot_util.human_format(422399296143, 2) == '422.40G') assert (plot_util.human_format(422399296143, 2, True) == '393.39Gi') -def test_time_format(): +def test_time_format() -> None: assert (plot_util.time_format(34) == '34s') assert (plot_util.time_format(59) == '59s') assert (plot_util.time_format(60) == '0:01') @@ -23,7 +23,7 @@ def test_time_format(): assert (plot_util.time_format(120) == '0:02') assert (plot_util.time_format(3694) == '1:01') -def test_split_path_prefix(): +def test_split_path_prefix() -> None: assert (plot_util.split_path_prefix( [] ) == ('', []) ) assert (plot_util.split_path_prefix([ '/a/0', '/b/1', '/c/2' ]) == @@ -31,7 +31,7 @@ def test_split_path_prefix(): assert ( plot_util.split_path_prefix([ '/a/b/0', '/a/b/1', '/a/b/2' ]) == ('/a/b', ['0', '1', '2']) ) -def test_columns(): +def test_columns() -> None: assert (plot_util.column_wrap(list(range(8)), 3, filler='--') == [ [ 0, 3, 6 ], [ 1, 4, 7 ], @@ -45,7 +45,7 @@ def test_columns(): [ 1 ], [ 2 ] ] ) -def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem): +def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: fs.create_file('/t/plot-k32-0.plot', st_size=108 * GB) fs.create_file('/t/plot-k32-1.plot', st_size=108 * GB) fs.create_file('/t/.plot-k32-2.plot', st_size=108 * GB) @@ -59,7 +59,7 @@ def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem): '/t/plot-k32-5.plot' ] ) -def test_get_plotsize(): +def test_get_plotsize() -> None: assert ( [659272492, 107287518791, 221143636517, 455373353413, 936816632588] == [plot_util.get_plotsize(n) for n in [25, 32, 33, 34, 35]] diff --git a/src/plotman/_tests/reporting_test.py b/src/plotman/_tests/reporting_test.py index 87c8a5e2..fcf7cab1 100644 --- a/src/plotman/_tests/reporting_test.py +++ b/src/plotman/_tests/reporting_test.py @@ -1,46 +1,47 @@ # TODO: migrate away from unittest patch import os -from unittest.mock import patch +import typing +from unittest.mock import patch, Mock from plotman import reporting from plotman import job -def test_phases_str_basic(): +def test_phases_str_basic() -> None: phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)]) assert reporting.phases_str(phases) == '1:2 2:3 3:4 4:0' -def test_phases_str_elipsis_1(): +def test_phases_str_elipsis_1() -> None: phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)]) assert reporting.phases_str(phases, 3) == '1:2 [+1] 3:4 4:0' -def test_phases_str_elipsis_2(): +def test_phases_str_elipsis_2() -> None: phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)]) assert reporting.phases_str(phases, 2) == '1:2 [+2] 4:0' -def test_phases_str_none(): +def test_phases_str_none() -> None: phases = job.Phase.list_from_tuples([(None, None), (3, 0)]) assert reporting.phases_str(phases) == '?:? 3:0' -def test_job_viz_empty(): +def test_job_viz_empty() -> None: assert(reporting.job_viz([]) == '1 2 3 4 ') @patch('plotman.job.Job') -def job_w_phase(ph, MockJob): +def job_w_phase(ph: typing.Tuple[typing.Optional[int], typing.Optional[int]], MockJob: Mock) -> Mock: j = MockJob() j.progress.return_value = job.Phase.from_tuple(ph) - return j + return j # type: ignore[no-any-return] -def test_job_viz_positions(): +def test_job_viz_positions() -> None: jobs = [job_w_phase((1, 1)), job_w_phase((2, 0)), job_w_phase((2, 4)), job_w_phase((2, 7)), job_w_phase((4, 0))] - assert(reporting.job_viz(jobs) == '1 . 2. . .3 4.') + assert(reporting.job_viz(jobs) == '1 . 2. . .3 4.') # type: ignore[arg-type] -def test_job_viz_counts(): +def test_job_viz_counts() -> None: jobs = [job_w_phase((2, 2)), job_w_phase((2, 3)), job_w_phase((2, 3)), @@ -59,4 +60,4 @@ def test_job_viz_counts(): job_w_phase((3, 1)), ] - assert(reporting.job_viz(jobs) == '1 2 .:;! 3 ! 4 ') + assert(reporting.job_viz(jobs) == '1 2 .:;! 3 ! 4 ') # type: ignore[arg-type] diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index 67073805..e210b26a 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -2,19 +2,20 @@ import re import statistics import sys +import typing import texttable as tt from plotman import plot_util -def analyze(logfilenames, clipterminals, bytmp, bybitfield): - data = {} +def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, bybitfield: bool) -> None: + data: typing.Dict[str, typing.Dict[str, typing.List[float]]] = {} for logfilename in logfilenames: with open(logfilename, 'r') as f: # Record of slicing and data associated with the slice sl = 'x' # Slice key - phase_time = {} # Map from phase index to time + phase_time: typing.Dict[str, float] = {} # Map from phase index to time n_sorts = 0 n_uniform = 0 is_first_last = False diff --git a/src/plotman/archive.py b/src/plotman/archive.py index 80d45e93..13c36ddd 100644 --- a/src/plotman/archive.py +++ b/src/plotman/archive.py @@ -8,6 +8,7 @@ import re import subprocess import sys +import typing from datetime import datetime import pendulum @@ -23,7 +24,7 @@ # TODO : write-protect and delete-protect archived plots -def spawn_archive_process(dir_cfg, arch_cfg, log_cfg, all_jobs): +def spawn_archive_process(dir_cfg: configuration.Directories, arch_cfg: configuration.Archiving, log_cfg: configuration.Logging, all_jobs: typing.List[job.Job]) -> typing.Tuple[typing.Union[bool, str, typing.Dict[str, object]], typing.List[str]]: '''Spawns a new archive process using the command created in the archive() function. Returns archiving status and a log message to print.''' @@ -32,7 +33,7 @@ def spawn_archive_process(dir_cfg, arch_cfg, log_cfg, all_jobs): # Look for running archive jobs. Be robust to finding more than one # even though the scheduler should only run one at a time. - arch_jobs = get_running_archive_jobs(arch_cfg) + arch_jobs: typing.List[typing.Union[int, str]] = [*get_running_archive_jobs(arch_cfg)] if not arch_jobs: (should_start, status_or_cmd, archive_log_messages) = archive(dir_cfg, arch_cfg, all_jobs) @@ -40,7 +41,7 @@ def spawn_archive_process(dir_cfg, arch_cfg, log_cfg, all_jobs): if not should_start: archiving_status = status_or_cmd else: - args = status_or_cmd + args: typing.Dict[str, object] = status_or_cmd # type: ignore[assignment] log_file_path = log_cfg.create_transfer_log_path(time=pendulum.now()) @@ -67,14 +68,19 @@ def spawn_archive_process(dir_cfg, arch_cfg, log_cfg, all_jobs): # of the log file will get closed explicitly while still # allowing handling of just the log file opening error. + if sys.platform == 'win32': + creationflags = subprocess.CREATE_NO_WINDOW + else: + creationflags = 0 + with open_log_file: # start_new_sessions to make the job independent of this controlling tty. - p = subprocess.Popen(**args, + p = subprocess.Popen(**args, # type: ignore[call-overload] shell=True, stdout=open_log_file, stderr=subprocess.STDOUT, start_new_session=True, - creationflags=0 if not _WINDOWS else subprocess.CREATE_NO_WINDOW) + creationflags=creationflags) # At least for now it seems that even if we get a new running # archive jobs list it doesn't contain the new rsync process. # My guess is that this is because the bash in the middle due to @@ -88,7 +94,7 @@ def spawn_archive_process(dir_cfg, arch_cfg, log_cfg, all_jobs): return archiving_status, log_messages -def compute_priority(phase, gb_free, n_plots): +def compute_priority(phase: job.Phase, gb_free: float, n_plots: int) -> int: # All these values are designed around dst buffer dirs of about # ~2TB size and containing k32 plots. TODO: Generalize, and # rewrite as a sort function. @@ -120,7 +126,7 @@ def compute_priority(phase, gb_free, n_plots): return priority -def get_archdir_freebytes(arch_cfg): +def get_archdir_freebytes(arch_cfg: configuration.Archiving) -> typing.Tuple[typing.Dict[str, int], typing.List[str]]: log_messages = [] target = arch_cfg.target_definition() @@ -128,7 +134,7 @@ def get_archdir_freebytes(arch_cfg): timeout = 5 try: completed_process = subprocess.run( - [target.disk_space_path], + [target.disk_space_path], # type: ignore[list-item] env={**os.environ, **arch_cfg.environment()}, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -171,7 +177,7 @@ def get_archdir_freebytes(arch_cfg): return archdir_freebytes, log_messages # TODO: maybe consolidate with similar code in job.py? -def get_running_archive_jobs(arch_cfg): +def get_running_archive_jobs(arch_cfg: configuration.Archiving) -> typing.List[int]: '''Look for running rsync jobs that seem to match the pattern we use for archiving them. Return a list of PIDs of matching jobs.''' jobs = [] @@ -189,12 +195,12 @@ def get_running_archive_jobs(arch_cfg): jobs.append(proc.pid) return jobs -def archive(dir_cfg, arch_cfg, all_jobs): +def archive(dir_cfg: configuration.Directories, arch_cfg: configuration.Archiving, all_jobs: typing.List[job.Job]) -> typing.Tuple[bool, typing.Optional[typing.Union[typing.Dict[str, object], str]], typing.List[str]]: '''Configure one archive job. Needs to know all jobs so it can avoid IO contention on the plotting dstdir drives. Returns either (False, ) if we should not execute an archive job or (True, ) with the archive command if we should.''' - log_messages = [] + log_messages: typing.List[str] = [] if arch_cfg is None: return (False, "No 'archive' settings declared in plotman.yaml", log_messages) @@ -244,7 +250,7 @@ def archive(dir_cfg, arch_cfg, all_jobs): source=chosen_plot, destination=archdir, ) - subprocess_arguments = { + subprocess_arguments: typing.Dict[str, object] = { 'args': arch_cfg.target_definition().transfer_path, 'env': {**os.environ, **env} } diff --git a/src/plotman/chia.py b/src/plotman/chia.py index 4ae3c25c..27cbddc2 100644 --- a/src/plotman/chia.py +++ b/src/plotman/chia.py @@ -1,14 +1,25 @@ +# mypy: allow_untyped_decorators + import functools +import typing import click from pathlib import Path +class CommandProtocol(typing.Protocol): + def make_context(self, info_name:str, args:typing.List[str]) -> click.Context: + ... + + def __call__(self) -> None: + ... + + class Commands: - def __init__(self): - self.by_version = {} + def __init__(self) -> None: + self.by_version: typing.Dict[typing.Sequence[int], CommandProtocol] = {} - def register(self, version): + def register(self, version: typing.Sequence[int]) -> typing.Callable[[CommandProtocol], None]: if version in self.by_version: raise Exception(f'Version already registered: {version!r}') if not isinstance(version, tuple): @@ -16,20 +27,19 @@ def register(self, version): return functools.partial(self._decorator, version=version) - def _decorator(self, command, *, version): + def _decorator(self, command: CommandProtocol, *, version: typing.Sequence[int]) -> None: self.by_version[version] = command # self.by_version = dict(sorted(self.by_version.items())) - def __getitem__(self, item): + def __getitem__(self, item: typing.Sequence[int]) -> typing.Callable[[], None]: return self.by_version[item] - def latest_command(self): + def latest_command(self) -> CommandProtocol: return max(self.by_version.items())[1] commands = Commands() - @commands.register(version=(1, 1, 2)) @click.command() # https://github.com/Chia-Network/chia-blockchain/blob/1.1.2/LICENSE @@ -81,7 +91,7 @@ def latest_command(self): "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True ) # end copied code -def _cli(): +def _cli_1_1_2() -> None: pass @@ -136,7 +146,7 @@ def _cli(): "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True ) # end copied code -def _cli(): +def _cli_1_1_3() -> None: pass @@ -191,7 +201,7 @@ def _cli(): "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True ) # end copied code -def _cli(): +def _cli_1_1_4() -> None: pass @@ -246,7 +256,7 @@ def _cli(): "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True ) # end copied code -def _cli(): +def _cli_1_1_5() -> None: pass @@ -301,7 +311,7 @@ def _cli(): "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True ) # end copied code -def _cli(): +def _cli_1_1_6() -> None: pass @@ -356,5 +366,5 @@ def _cli(): "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True ) # end copied code -def _cli(): +def _cli_1_1_7() -> None: pass diff --git a/src/plotman/chiapos.py b/src/plotman/chiapos.py index 553dbcab..6fb55f34 100644 --- a/src/plotman/chiapos.py +++ b/src/plotman/chiapos.py @@ -60,7 +60,7 @@ # https://github.com/Chia-Network/chiapos/blob/1.0.2/LICENSE # https://github.com/Chia-Network/chiapos/blob/1.0.2/src/util.hpp # start ported code -def ByteAlign(num_bits): +def ByteAlign(num_bits: float) -> float: return (num_bits + (8 - ((num_bits) % 8)) % 8) # end ported code @@ -68,20 +68,20 @@ def ByteAlign(num_bits): # https://github.com/Chia-Network/chiapos/blob/1.0.2/LICENSE # https://github.com/Chia-Network/chiapos/blob/1.0.2/src/entry_sizes.hpp # start ported code -def CalculateLinePointSize(k): +def CalculateLinePointSize(k: int) -> float: return ByteAlign(2 * k) / 8 # This is the full size of the deltas section in a park. However, it will not be fully filled -def CalculateMaxDeltasSize(k, table_index): +def CalculateMaxDeltasSize(k: int, table_index: int) -> float: if (table_index == 1): return ByteAlign((kEntriesPerPark - 1) * kMaxAverageDeltaTable1) / 8 return ByteAlign((kEntriesPerPark - 1) * kMaxAverageDelta) / 8 -def CalculateStubsSize(k): +def CalculateStubsSize(k: int) -> float: return ByteAlign((kEntriesPerPark - 1) * (k - kStubMinusBits)) / 8 -def CalculateParkSize(k, table_index): +def CalculateParkSize(k: int, table_index: int) -> float: return CalculateLinePointSize(k) + CalculateStubsSize(k) + CalculateMaxDeltasSize(k, table_index); # end ported code diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 220dc538..33f8589e 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -4,12 +4,13 @@ import stat import tempfile import textwrap -from typing import Dict, List, Optional +from typing import Dict, Generator, List, Mapping, Optional import appdirs import attr import desert import marshmallow +import pendulum import yaml from plotman import resources as plotman_resources @@ -19,12 +20,13 @@ class ConfigurationException(Exception): """Raised when plotman.yaml configuration is missing or malformed.""" -def get_path(): +def get_path() -> str: """Return path to where plotman.yaml configuration file should exist.""" - return appdirs.user_config_dir("plotman") + "/plotman.yaml" + config_dir: str = appdirs.user_config_dir("plotman") + return config_dir + "/plotman.yaml" -def read_configuration_text(config_path): +def read_configuration_text(config_path: str) -> str: try: with open(config_path, "r") as file: return file.read() @@ -35,7 +37,7 @@ def read_configuration_text(config_path): ) from e -def get_validated_configs(config_text, config_path, preset_target_definitions_text): +def get_validated_configs(config_text: str, config_path: str, preset_target_definitions_text: str) -> "PlotmanConfig": """Return a validated instance of PlotmanConfig with data from plotman.yaml :raises ConfigurationException: Raised when plotman.yaml is either missing or malformed @@ -55,6 +57,7 @@ def get_validated_configs(config_text, config_path, preset_target_definitions_te raise Exception(message) + loaded: PlotmanConfig try: loaded = schema.load(config_objects) except marshmallow.exceptions.ValidationError as e: @@ -75,11 +78,11 @@ def get_validated_configs(config_text, config_path, preset_target_definitions_te return loaded class CustomStringField(marshmallow.fields.String): - def _deserialize(self, value, attr, data, **kwargs): + def _deserialize(self, value: object, attr: Optional[str], data: Optional[Mapping[str, object]], **kwargs: Dict[str, object]) -> str: if isinstance(value, int): value = str(value) - return super()._deserialize(value, attr, data, **kwargs) + return super()._deserialize(value, attr, data, **kwargs) # type: ignore[no-any-return] # Data models used to deserializing/formatting plotman.yaml files. @@ -120,27 +123,28 @@ class Archiving: index: int = 0 # If not explicit, "index" will default to 0 target_definitions: Dict[str, ArchivingTarget] = attr.ib(factory=dict) - def target_definition(self): + def target_definition(self) -> ArchivingTarget: return self.target_definitions[self.target] def environment( self, - source=None, - destination=None, - ): + source: Optional[str] = None, + destination: Optional[str] = None, + ) -> Dict[str, str]: target = self.target_definition() - complete = {**target.env, **self.env} + maybe_complete = {**target.env, **self.env} - missing_mandatory_keys = [ - key - for key, value in complete.items() - if value is None - ] + complete = { + key: value + for key, value in maybe_complete.items() + if value is not None + } - if len(missing_mandatory_keys) > 0: - target = repr(self.target) + if len(complete) != len(maybe_complete): + missing_mandatory_keys = sorted(maybe_complete.keys() - complete.keys()) + target_repr = repr(self.target) missing = ', '.join(repr(key) for key in missing_mandatory_keys) - message = f'Missing env options for archival target {target}: {missing}' + message = f'Missing env options for archival target {target_repr}: {missing}' raise Exception(message) variables = {**os.environ, **complete} @@ -154,11 +158,14 @@ def environment( return complete - def maybe_create_scripts(self, temp): + def maybe_create_scripts(self, temp: str) -> None: rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR target = self.target_definition() if target.disk_space_path is None: + if target.disk_space_script is None: + raise Exception(f"One of `disk_space_path` or `disk_space_script` must be specified. Using target {self.target!r}") + with tempfile.NamedTemporaryFile( mode='w', encoding='utf-8', @@ -172,6 +179,9 @@ def maybe_create_scripts(self, temp): os.chmod(target.disk_space_path, rwx) if target.transfer_path is None: + if target.transfer_script is None: + raise Exception(f"One of `transfer_path` or `transfer_script` must be specified. Using target {self.target!r}") + with tempfile.NamedTemporaryFile( mode='w', encoding='utf-8', @@ -194,26 +204,26 @@ class Logging: transfers: str = os.path.join(appdirs.user_data_dir("plotman"), 'transfers') application: str = os.path.join(appdirs.user_log_dir("plotman"), 'plotman.log') - def setup(self): + def setup(self) -> None: os.makedirs(self.plots, exist_ok=True) os.makedirs(self.transfers, exist_ok=True) os.makedirs(os.path.dirname(self.application), exist_ok=True) - def create_plot_log_path(self, time): + def create_plot_log_path(self, time: pendulum.DateTime) -> str: return self._create_log_path( time=time, directory=self.plots, group='plot', ) - def create_transfer_log_path(self, time): + def create_transfer_log_path(self, time: pendulum.DateTime) -> str: return self._create_log_path( time=time, directory=self.transfers, group='transfer', ) - def _create_log_path(self, time, directory, group): + def _create_log_path(self, time: pendulum.DateTime, directory: str, group: str) -> str: timestamp = time.isoformat(timespec='microseconds').replace(':', '_') return os.path.join(directory, f'{timestamp}.{group}.log') @@ -224,22 +234,22 @@ class Directories: tmp2: Optional[str] = None tmp_overrides: Optional[Dict[str, TmpOverrides]] = None - def dst_is_tmp(self): + def dst_is_tmp(self) -> bool: return self.dst is None and self.tmp2 is None - def dst_is_tmp2(self): + def dst_is_tmp2(self) -> bool: return self.dst is None and self.tmp2 is not None - def get_dst_directories(self): + def get_dst_directories(self) -> List[str]: """Returns either or . If Directories.dst is None, Use Directories.tmp as dst directory. """ if self.dst_is_tmp2(): - return [self.tmp2] + return [self.tmp2] # type: ignore[list-item] elif self.dst_is_tmp(): return self.tmp - return self.dst + return self.dst # type: ignore[return-value] @attr.frozen class Scheduling: @@ -288,7 +298,7 @@ class PlotmanConfig: version: List[int] = [0] @contextlib.contextmanager - def setup(self): + def setup(self) -> Generator[None, None, None]: prefix = f'plotman-pid_{os.getpid()}-' self.logging.setup() diff --git a/src/plotman/csv_exporter.py b/src/plotman/csv_exporter.py index 0a1b5ade..4cb679e0 100644 --- a/src/plotman/csv_exporter.py +++ b/src/plotman/csv_exporter.py @@ -1,61 +1,65 @@ import csv import sys -from dateutil.parser import parse as parse_date +import typing import attr +import attr._make +import pendulum from plotman.log_parser import PlotLogParser +import plotman.plotinfo -def row_ib(name): - return attr.ib(converter=str, metadata={'name': name}) @attr.frozen class Row: - plot_id: str = row_ib(name='Plot ID') - started_at: str = row_ib(name='Started at') - date: str = row_ib(name='Date') - size: str = row_ib(name='Size') - buffer: str = row_ib(name='Buffer') - buckets: str = row_ib(name='Buckets') - threads: str = row_ib(name='Threads') - tmp_dir_1: str = row_ib(name='Tmp dir 1') - tmp_dir_2: str = row_ib(name='Tmp dir 2') - phase_1_duration_raw: str = row_ib(name='Phase 1 duration (raw)') - phase_1_duration: str = row_ib(name='Phase 1 duration') - phase_1_duration_minutes: str = row_ib(name='Phase 1 duration (minutes)') - phase_1_duration_hours: str = row_ib(name='Phase 1 duration (hours)') - phase_2_duration_raw: str = row_ib(name='Phase 2 duration (raw)') - phase_2_duration: str = row_ib(name='Phase 2 duration') - phase_2_duration_minutes: str = row_ib(name='Phase 2 duration (minutes)') - phase_2_duration_hours: str = row_ib(name='Phase 2 duration (hours)') - phase_3_duration_raw: str = row_ib(name='Phase 3 duration (raw)') - phase_3_duration: str = row_ib(name='Phase 3 duration') - phase_3_duration_minutes: str = row_ib(name='Phase 3 duration (minutes)') - phase_3_duration_hours: str = row_ib(name='Phase 3 duration (hours)') - phase_4_duration_raw: str = row_ib(name='Phase 4 duration (raw)') - phase_4_duration: str = row_ib(name='Phase 4 duration') - phase_4_duration_minutes: str = row_ib(name='Phase 4 duration (minutes)') - phase_4_duration_hours: str = row_ib(name='Phase 4 duration (hours)') - total_time_raw: str = row_ib(name='Total time (raw)') - total_time: str = row_ib(name='Total time') - total_time_minutes: str = row_ib(name='Total time (minutes)') - total_time_hours: str = row_ib(name='Total time (hours)') - copy_time_raw: str = row_ib(name='Copy time (raw)') - copy_time: str = row_ib(name='Copy time') - copy_time_minutes: str = row_ib(name='Copy time (minutes)') - copy_time_hours: str = row_ib(name='Copy time (hours)') - filename: str = row_ib(name='Filename') + plot_id: str = attr.ib(converter=str, metadata={'name': 'Plot ID'}) + started_at: str = attr.ib(converter=str, metadata={'name': 'Started at'}) + date: str = attr.ib(converter=str, metadata={'name': 'Date'}) + size: str = attr.ib(converter=str, metadata={'name': 'Size'}) + buffer: str = attr.ib(converter=str, metadata={'name': 'Buffer'}) + buckets: str = attr.ib(converter=str, metadata={'name': 'Buckets'}) + threads: str = attr.ib(converter=str, metadata={'name': 'Threads'}) + tmp_dir_1: str = attr.ib(converter=str, metadata={'name': 'Tmp dir 1'}) + tmp_dir_2: str = attr.ib(converter=str, metadata={'name': 'Tmp dir 2'}) + phase_1_duration_raw: str = attr.ib(converter=str, metadata={'name': 'Phase 1 duration (raw)'}) + phase_1_duration: str = attr.ib(converter=str, metadata={'name': 'Phase 1 duration'}) + phase_1_duration_minutes: str = attr.ib(converter=str, metadata={'name': 'Phase 1 duration (minutes)'}) + phase_1_duration_hours: str = attr.ib(converter=str, metadata={'name': 'Phase 1 duration (hours)'}) + phase_2_duration_raw: str = attr.ib(converter=str, metadata={'name': 'Phase 2 duration (raw)'}) + phase_2_duration: str = attr.ib(converter=str, metadata={'name': 'Phase 2 duration'}) + phase_2_duration_minutes: str = attr.ib(converter=str, metadata={'name': 'Phase 2 duration (minutes)'}) + phase_2_duration_hours: str = attr.ib(converter=str, metadata={'name': 'Phase 2 duration (hours)'}) + phase_3_duration_raw: str = attr.ib(converter=str, metadata={'name': 'Phase 3 duration (raw)'}) + phase_3_duration: str = attr.ib(converter=str, metadata={'name': 'Phase 3 duration'}) + phase_3_duration_minutes: str = attr.ib(converter=str, metadata={'name': 'Phase 3 duration (minutes)'}) + phase_3_duration_hours: str = attr.ib(converter=str, metadata={'name': 'Phase 3 duration (hours)'}) + phase_4_duration_raw: str = attr.ib(converter=str, metadata={'name': 'Phase 4 duration (raw)'}) + phase_4_duration: str = attr.ib(converter=str, metadata={'name': 'Phase 4 duration'}) + phase_4_duration_minutes: str = attr.ib(converter=str, metadata={'name': 'Phase 4 duration (minutes)'}) + phase_4_duration_hours: str = attr.ib(converter=str, metadata={'name': 'Phase 4 duration (hours)'}) + total_time_raw: str = attr.ib(converter=str, metadata={'name': 'Total time (raw)'}) + total_time: str = attr.ib(converter=str, metadata={'name': 'Total time'}) + total_time_minutes: str = attr.ib(converter=str, metadata={'name': 'Total time (minutes)'}) + total_time_hours: str = attr.ib(converter=str, metadata={'name': 'Total time (hours)'}) + copy_time_raw: str = attr.ib(converter=str, metadata={'name': 'Copy time (raw)'}) + copy_time: str = attr.ib(converter=str, metadata={'name': 'Copy time'}) + copy_time_minutes: str = attr.ib(converter=str, metadata={'name': 'Copy time (minutes)'}) + copy_time_hours: str = attr.ib(converter=str, metadata={'name': 'Copy time (hours)'}) + filename: str = attr.ib(converter=str, metadata={'name': 'Filename'}) @classmethod - def names(cls): + def names(cls) -> typing.List[str]: return [field.metadata['name'] for field in attr.fields(cls)] @classmethod - def from_info(cls, info): + def from_info(cls, info: plotman.plotinfo.PlotInfo) -> "Row": + if info.started_at is None: + raise Exception(f'Unexpected None start time for file: {info.filename}') + return cls( plot_id=info.plot_id, started_at=info.started_at.isoformat(), - date=info.started_at.date().isoformat(), + date=info.started_at.date().isoformat(), # type: ignore[no-untyped-call] size=info.plot_size, buffer=info.buffer, buckets=info.buckets, @@ -89,13 +93,21 @@ def from_info(cls, info): filename=info.filename, ) - def name_dict(self): + def name_dict(self) -> typing.Dict[str, object]: return { field.metadata['name']: value for field, value in zip(attr.fields(type(self)), attr.astuple(self)) } -def parse_logs(logfilenames): + +def key_on_plot_info_started_at(element: plotman.plotinfo.PlotInfo) -> pendulum.DateTime: + if element.started_at is None: + return pendulum.now().add(years=9999) + + return element.started_at + + +def parse_logs(logfilenames: typing.Sequence[str]) -> typing.List[plotman.plotinfo.PlotInfo]: parser = PlotLogParser() result = [] @@ -106,11 +118,12 @@ def parse_logs(logfilenames): if not info.in_progress(): result.append(info) - result.sort(key=lambda element: element.started_at) + result.sort(key=key_on_plot_info_started_at) + return result -def generate(logfilenames, file): +def generate(logfilenames: typing.List[str], file: typing.TextIO) -> None: writer = csv.DictWriter(file, fieldnames=Row.names()) writer.writeheader() diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py index 18a1bb29..82ea89a8 100644 --- a/src/plotman/interactive.py +++ b/src/plotman/interactive.py @@ -5,6 +5,8 @@ import os import subprocess import sys +import typing + from plotman import archive, configuration, manager, reporting from plotman.job import Job @@ -13,47 +15,50 @@ class TerminalTooSmallError(Exception): pass class Log: - def __init__(self): + entries: typing.List[str] + cur_pos: int + + def __init__(self) -> None: self.entries = [] self.cur_pos = 0 # TODO: store timestamp as actual timestamp indexing the messages - def log(self, msg): + def log(self, msg: str) -> None: '''Log the message and scroll to the end of the log''' ts = datetime.datetime.now().strftime('%m-%d %H:%M:%S') self.entries.append(ts + ' ' + msg) self.cur_pos = len(self.entries) - def tail(self, num_entries): + def tail(self, num_entries: int) -> typing.List[str]: '''Return the entries at the end of the log. Consider cur_slice() instead.''' return self.entries[-num_entries:] - def shift_slice(self, offset): + def shift_slice(self, offset: int) -> None: '''Positive shifts towards end, negative shifts towards beginning''' self.cur_pos = max(0, min(len(self.entries), self.cur_pos + offset)) - def shift_slice_to_end(self): + def shift_slice_to_end(self) -> None: self.cur_pos = len(self.entries) - def get_cur_pos(self): + def get_cur_pos(self) -> int: return self.cur_pos - def cur_slice(self, num_entries): + def cur_slice(self, num_entries: int) -> typing.List[str]: '''Return num_entries log entries up to the current slice position''' return self.entries[max(0, self.cur_pos - num_entries) : self.cur_pos] - def fill_log(self): + def fill_log(self) -> None: '''Add a bunch of stuff to the log. Useful for testing.''' for i in range(100): self.log('Log line %d' % i) -def plotting_status_msg(active, status): +def plotting_status_msg(active: bool, status: str) -> str: if active: return '(active) ' + status else: return '(inactive) ' + status -def archiving_status_msg(configured, active, status): +def archiving_status_msg(configured: bool, active: bool, status: str) -> str: if configured: if active: return '(active) ' + status @@ -63,7 +68,7 @@ def archiving_status_msg(configured, active, status): return '(not configured)' # cmd_autostart_plotting is the (optional) argument passed from the command line. May be None -def curses_main(stdscr, cmd_autostart_plotting, cmd_autostart_archiving, cfg): +def curses_main(stdscr: typing.Any, cmd_autostart_plotting: typing.Optional[bool], cmd_autostart_archiving: typing.Optional[bool], cfg: configuration.PlotmanConfig) -> None: log = Log() if cmd_autostart_plotting is not None: @@ -81,7 +86,7 @@ def curses_main(stdscr, cmd_autostart_plotting, cmd_autostart_archiving, cfg): archiving_active = cfg.commands.interactive.autostart_archiving plotting_status = '' # todo rename these msg? - archiving_status = '' + archiving_status: typing.Union[bool, str, typing.Dict[str, object]] = '' stdscr.nodelay(True) # make getch() non-blocking stdscr.timeout(2000) @@ -137,7 +142,7 @@ def curses_main(stdscr, cmd_autostart_plotting, cmd_autostart_archiving, cfg): aging_reason = msg plotting_status = msg - if archiving_configured: + if cfg.archiving is not None: if archiving_active: archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, cfg.logging, jobs) for log_message in log_messages: @@ -179,8 +184,8 @@ def curses_main(stdscr, cmd_autostart_plotting, cmd_autostart_archiving, cfg): tmp_prefix = os.path.commonpath(cfg.directories.tmp) dst_dir = cfg.directories.get_dst_directories() dst_prefix = os.path.commonpath(dst_dir) - if archiving_configured: - archive_directories = archdir_freebytes.keys() + if archdir_freebytes is not None: + archive_directories = list(archdir_freebytes.keys()) if len(archive_directories) == 0: arch_prefix = '' else: @@ -193,7 +198,7 @@ def curses_main(stdscr, cmd_autostart_plotting, cmd_autostart_archiving, cfg): jobs, cfg.directories, cfg.scheduling, n_cols, 0, n_tmpdirs, tmp_prefix) dst_report = reporting.dst_dir_report( jobs, dst_dir, n_cols, dst_prefix) - if archiving_configured: + if archdir_freebytes is not None: arch_report = reporting.arch_dir_report(archdir_freebytes, n_cols, arch_prefix) if not arch_report: arch_report = '' @@ -250,7 +255,7 @@ def curses_main(stdscr, cmd_autostart_plotting, cmd_autostart_archiving, cfg): header_win.addnstr(' rchival: ', linecap, curses.A_BOLD) header_win.addnstr( archiving_status_msg(archiving_configured, - archiving_active, archiving_status), linecap) + archiving_active, archiving_status), linecap) # type: ignore[arg-type] # Oneliner progress display header_win.addnstr(1, 0, 'Jobs (%d): ' % len(jobs), linecap) @@ -338,7 +343,7 @@ def curses_main(stdscr, cmd_autostart_plotting, cmd_autostart_archiving, cfg): else: pressed_key = key -def run_interactive(cfg, autostart_plotting=None, autostart_archiving=None): +def run_interactive(cfg: configuration.PlotmanConfig, autostart_plotting: typing.Optional[bool] = None, autostart_archiving: typing.Optional[bool] = None) -> None: locale.setlocale(locale.LC_ALL, '') code = locale.getpreferredencoding() # Then use code as the encoding for str.encode() calls. diff --git a/src/plotman/job.py b/src/plotman/job.py index e99ac07f..6f19c755 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -11,6 +11,7 @@ from datetime import datetime from enum import Enum, auto from subprocess import call +import typing import attr import click @@ -20,15 +21,15 @@ from plotman import chia -def job_phases_for_tmpdir(d, all_jobs): +def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: '''Return phase 2-tuples for jobs running on tmpdir d''' return sorted([j.progress() for j in all_jobs if j.tmpdir == d]) -def job_phases_for_dstdir(d, all_jobs): +def job_phases_for_dstdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: '''Return phase 2-tuples for jobs outputting to dstdir d''' return sorted([j.progress() for j in all_jobs if j.dstdir == d]) -def is_plotting_cmdline(cmdline): +def is_plotting_cmdline(cmdline: typing.List[str]) -> bool: if cmdline and 'python' in cmdline[0].lower(): cmdline = cmdline[1:] return ( @@ -38,11 +39,15 @@ def is_plotting_cmdline(cmdline): and 'create' == cmdline[2] ) -def parse_chia_plot_time(s): +def parse_chia_plot_time(s: str) -> pendulum.DateTime: # This will grow to try ISO8601 as well for when Chia logs that way - return pendulum.from_format(s, 'ddd MMM DD HH:mm:ss YYYY', locale='en', tz=None) + # TODO: unignore once fixed upstream + # https://github.com/sdispater/pendulum/pull/548 + return pendulum.from_format(s, 'ddd MMM DD HH:mm:ss YYYY', locale='en', tz=None) # type: ignore[arg-type] -def parse_chia_plots_create_command_line(command_line): +def parse_chia_plots_create_command_line( + command_line: typing.List[str], +) -> "ParsedChiaPlotsCreateCommand": command_line = list(command_line) # Parse command line args if 'python' in command_line[0].lower(): @@ -84,7 +89,12 @@ def parse_chia_plots_create_command_line(command_line): ) class ParsedChiaPlotsCreateCommand: - def __init__(self, error, help, parameters): + def __init__( + self, + error: click.ClickException, + help: bool, + parameters: typing.Dict[str, object], + ) -> None: self.error = error self.help = help self.parameters = parameters @@ -96,14 +106,14 @@ class Phase: minor: int = 0 known: bool = True - def __lt__(self, other): + def __lt__(self, other: "Phase") -> bool: return ( (not self.known, self.major, self.minor) < (not other.known, other.major, other.minor) ) @classmethod - def from_tuple(cls, t): + def from_tuple(cls, t: typing.Tuple[typing.Optional[int], typing.Optional[int]]) -> "Phase": if len(t) != 2: raise Exception(f'phase must be created from 2-tuple: {t!r}') @@ -113,10 +123,13 @@ def from_tuple(cls, t): if t[0] is None: return cls(known=False) - return cls(major=t[0], minor=t[1]) + return cls(major=t[0], minor=t[1]) # type: ignore[arg-type] @classmethod - def list_from_tuples(cls, l): + def list_from_tuples( + cls, + l: typing.Sequence[typing.Tuple[typing.Optional[int], typing.Optional[int]]], + ) -> typing.List["Phase"]: return [cls.from_tuple(t) for t in l] # TODO: be more principled and explicit about what we cache vs. what we look up @@ -124,17 +137,30 @@ def list_from_tuples(cls, l): class Job: 'Represents a plotter job' - logfile = '' - jobfile = '' - job_id = 0 - plot_id = '--------' - proc = None # will get a psutil.Process + logfile: str = '' + jobfile: str = '' + job_id: int = 0 + plot_id: str = '--------' + proc: psutil.Process + k: int + r: int + u: int + b: int + n: int + tmpdir: str + tmp2dir: str + dstdir: str - def get_running_jobs(logroot, cached_jobs=()): + @classmethod + def get_running_jobs( + cls, + logroot: str, + cached_jobs: typing.Sequence["Job"] = (), + ) -> typing.List["Job"]: '''Return a list of running plot jobs. If a cache of preexisting jobs is provided, reuse those previous jobs without updating their information. Always look for new jobs not already in the cache.''' - jobs = [] + jobs: typing.List[Job] = [] cached_jobs_by_pid = { j.proc.pid: j for j in cached_jobs } with contextlib.ExitStack() as exit_stack: @@ -183,7 +209,7 @@ def get_running_jobs(logroot, cached_jobs=()): ) if parsed_command.error is not None: continue - job = Job( + job = cls( proc=proc, parsed_command=parsed_command, logroot=logroot, @@ -195,7 +221,12 @@ def get_running_jobs(logroot, cached_jobs=()): return jobs - def __init__(self, proc, parsed_command, logroot): + def __init__( + self, + proc: psutil.Process, + parsed_command: ParsedChiaPlotsCreateCommand, + logroot: str, + ) -> None: '''Initialize from an existing psutil.Process object. must know logroot in order to understand open files''' self.proc = proc # These are dynamic, cached, and need to be udpated periodically @@ -225,16 +256,16 @@ def __init__(self, proc, parsed_command, logroot): # 'exclude_final_dir': False, # } - self.k = self.args['size'] - self.r = self.args['num_threads'] - self.u = self.args['buckets'] - self.b = self.args['buffer'] - self.n = self.args['num'] - self.tmpdir = self.args['tmp_dir'] - self.tmp2dir = self.args['tmp2_dir'] - self.dstdir = self.args['final_dir'] + self.k = self.args['size'] # type: ignore[assignment] + self.r = self.args['num_threads'] # type: ignore[assignment] + self.u = self.args['buckets'] # type: ignore[assignment] + self.b = self.args['buffer'] # type: ignore[assignment] + self.n = self.args['num'] # type: ignore[assignment] + self.tmpdir = self.args['tmp_dir'] # type: ignore[assignment] + self.tmp2dir = self.args['tmp2_dir'] # type: ignore[assignment] + self.dstdir = self.args['final_dir'] # type: ignore[assignment] - plot_cwd = self.proc.cwd() + plot_cwd: str = self.proc.cwd() self.tmpdir = os.path.join(plot_cwd, self.tmpdir) if self.tmp2dir is not None: self.tmp2dir = os.path.join(plot_cwd, self.tmp2dir) @@ -262,7 +293,7 @@ def __init__(self, proc, parsed_command, logroot): - def init_from_logfile(self): + def init_from_logfile(self) -> None: '''Read plot ID and job start time from logfile. Return true if we find all the info as expected, false otherwise''' assert self.logfile @@ -295,15 +326,15 @@ def init_from_logfile(self): # TODO: we never come back to this; e.g. plot_id may remain uninitialized. # TODO: should we just use the process start time instead? if not found_log: - self.start_time = datetime.fromtimestamp(os.path.getctime(self.logfile)) + self.start_time = pendulum.from_timestamp(os.path.getctime(self.logfile)) # Load things from logfile that are dynamic self.update_from_logfile() - def update_from_logfile(self): + def update_from_logfile(self) -> None: self.set_phase_from_logfile() - def set_phase_from_logfile(self): + def set_phase_from_logfile(self) -> None: assert self.logfile # Map from phase number to subphase number reached in that phase. @@ -355,15 +386,15 @@ def set_phase_from_logfile(self): else: self.phase = Phase(major=0, minor=0) - def progress(self): + def progress(self) -> Phase: '''Return a 2-tuple with the job phase and subphase (by reading the logfile)''' return self.phase - def plot_id_prefix(self): + def plot_id_prefix(self) -> str: return self.plot_id[:8] # TODO: make this more useful and complete, and/or make it configurable - def status_str_long(self): + def status_str_long(self) -> str: return '{plot_id}\nk={k} r={r} b={b} u={u}\npid:{pid}\ntmp:{tmp}\ntmp2:{tmp2}\ndst:{dst}\nlogfile:{logfile}'.format( plot_id = self.plot_id, k = self.k, @@ -374,14 +405,14 @@ def status_str_long(self): tmp = self.tmpdir, tmp2 = self.tmp2dir, dst = self.dstdir, - plotid = self.plot_id, logfile = self.logfile ) - def get_mem_usage(self): - return self.proc.memory_info().vms # Total, inc swapped + def get_mem_usage(self) -> int: + # Total, inc swapped + return self.proc.memory_info().vms # type: ignore[no-any-return] - def get_tmp_usage(self): + def get_tmp_usage(self) -> int: total_bytes = 0 with contextlib.suppress(FileNotFoundError): # The directory might not exist at this name, or at all, anymore @@ -393,7 +424,7 @@ def get_tmp_usage(self): total_bytes += entry.stat().st_size return total_bytes - def get_run_status(self): + def get_run_status(self) -> str: '''Running, suspended, etc.''' status = self.proc.status() if status == psutil.STATUS_RUNNING: @@ -405,19 +436,19 @@ def get_run_status(self): elif status == psutil.STATUS_STOPPED: return 'STP' else: - return self.proc.status() + return self.proc.status() # type: ignore[no-any-return] - def get_time_wall(self): + def get_time_wall(self) -> int: create_time = datetime.fromtimestamp(self.proc.create_time()) return int((datetime.now() - create_time).total_seconds()) - def get_time_user(self): + def get_time_user(self) -> int: return int(self.proc.cpu_times().user) - def get_time_sys(self): + def get_time_sys(self) -> int: return int(self.proc.cpu_times().system) - def get_time_iowait(self): + def get_time_iowait(self) -> typing.Optional[int]: cpu_times = self.proc.cpu_times() iowait = getattr(cpu_times, 'iowait', None) if iowait is None: @@ -425,14 +456,14 @@ def get_time_iowait(self): return int(iowait) - def suspend(self, reason=''): + def suspend(self, reason: str = '') -> None: self.proc.suspend() self.status_note = reason - def resume(self): + def resume(self) -> None: self.proc.resume() - def get_temp_files(self): + def get_temp_files(self) -> typing.Set[str]: # Prevent duplicate file paths by using set. temp_files = set([]) for f in self.proc.open_files(): @@ -444,7 +475,7 @@ def get_temp_files(self): temp_files.add(f.path) return temp_files - def cancel(self): + def cancel(self) -> None: 'Cancel an already running job' # We typically suspend the job as the first action in killing it, so it # doesn't create more tmp files during death. However, terminate() won't diff --git a/src/plotman/log_parser.py b/src/plotman/log_parser.py index c125c584..5c9eac19 100644 --- a/src/plotman/log_parser.py +++ b/src/plotman/log_parser.py @@ -1,5 +1,7 @@ import os import re +import typing + from plotman.plotinfo import PlotInfo import plotman.job @@ -7,7 +9,7 @@ class PlotLogParser: """Parser for a finished plotting job""" - def parse(self, file) -> PlotInfo: + def parse(self, file: typing.TextIO) -> PlotInfo: """Parses a single log and returns its info""" entry = PlotInfo() @@ -139,4 +141,5 @@ def plot_start_date(self, line: str, entry: PlotInfo) -> bool: # Ignore lines starting with Bucket # Bucket 0 uniform sort. Ram: 3.250GiB, u_sort min: 0.563GiB, qs min: 0.281GiB. def ignore_line(self, line: str, _: PlotInfo) -> bool: - return re.search(r'^\tBucket', line) \ No newline at end of file + m = re.search(r'^\tBucket', line) + return m != None \ No newline at end of file diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 7c5b116d..e8105aab 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -6,6 +6,7 @@ import subprocess import sys import time +import typing from datetime import datetime import pendulum @@ -15,6 +16,7 @@ from plotman import \ archive # for get_archdir_freebytes(). TODO: move to avoid import loop from plotman import job, plot_util +import plotman.configuration # Constants MIN = 60 # Seconds @@ -22,21 +24,19 @@ MAX_AGE = 1000_000_000 # Arbitrary large number of seconds -_WINDOWS = sys.platform == 'win32' - -def dstdirs_to_furthest_phase(all_jobs): +def dstdirs_to_furthest_phase(all_jobs: typing.List[job.Job]) -> typing.Dict[str, job.Phase]: '''Return a map from dst dir to a phase tuple for the most progressed job that is emitting to that dst dir.''' - result = {} + result: typing.Dict[str, job.Phase] = {} for j in all_jobs: if not j.dstdir in result.keys() or result[j.dstdir] < j.progress(): result[j.dstdir] = j.progress() return result -def dstdirs_to_youngest_phase(all_jobs): +def dstdirs_to_youngest_phase(all_jobs: typing.List[job.Job]) -> typing.Dict[str, job.Phase]: '''Return a map from dst dir to a phase tuple for the least progressed job that is emitting to that dst dir.''' - result = {} + result: typing.Dict[str, job.Phase] = {} for j in all_jobs: if j.dstdir is None: continue @@ -44,7 +44,7 @@ def dstdirs_to_youngest_phase(all_jobs): result[j.dstdir] = j.progress() return result -def phases_permit_new_job(phases, d, sched_cfg, dir_cfg): +def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plotman.configuration.Scheduling, dir_cfg: plotman.configuration.Directories) -> bool: '''Scheduling logic: return True if it's OK to start a new job on a tmp dir with existing jobs in the provided phases.''' # Filter unknown-phase jobs @@ -73,7 +73,7 @@ def phases_permit_new_job(phases, d, sched_cfg, dir_cfg): return True -def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg, log_cfg): +def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: plotman.configuration.Scheduling, plotting_cfg: plotman.configuration.Plotting, log_cfg: plotman.configuration.Logging) -> typing.Tuple[bool, str]: jobs = job.Job.get_running_jobs(log_cfg.plots) wait_reason = None # If we don't start a job this iteration, this says why. @@ -97,8 +97,9 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg, log_cfg): # Plot to oldest tmpdir. tmpdir = max(rankable, key=operator.itemgetter(1))[0] + dstdir: str if dir_cfg.dst_is_tmp2(): - dstdir = dir_cfg.tmp2 + dstdir = dir_cfg.tmp2 # type: ignore[assignment] elif dir_cfg.dst_is_tmp(): dstdir = tmpdir else: @@ -111,11 +112,13 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg, log_cfg): if unused_dirs: dstdir = random.choice(unused_dirs) else: - dstdir = max(dir2ph, key=dir2ph.get) + def key(key: str) -> job.Phase: + return dir2ph[key] + dstdir = max(dir2ph, key=key) log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) - plot_args = ['chia', 'plots', 'create', + plot_args: typing.List[str] = ['chia', 'plots', 'create', '-k', str(plotting_cfg.k), '-r', str(plotting_cfg.n_threads), '-u', str(plotting_cfg.n_buckets), @@ -169,6 +172,13 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg, log_cfg): # of the log file will get closed explicitly while still # allowing handling of just the log file opening error. + if sys.platform == 'win32': + creationflags = subprocess.CREATE_NO_WINDOW + nice = psutil.BELOW_NORMAL_PRIORITY_CLASS + else: + creationflags = 0 + nice = 15 + with open_log_file: # start_new_sessions to make the job independent of this controlling tty (POSIX only). # subprocess.CREATE_NO_WINDOW to make the process independent of this controlling tty and have no console window on Windows. @@ -176,14 +186,14 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg, log_cfg): stdout=open_log_file, stderr=subprocess.STDOUT, start_new_session=True, - creationflags=0 if not _WINDOWS else subprocess.CREATE_NO_WINDOW) + creationflags=creationflags) - psutil.Process(p.pid).nice(15 if not _WINDOWS else psutil.BELOW_NORMAL_PRIORITY_CLASS) + psutil.Process(p.pid).nice(nice) return (True, logmsg) return (False, wait_reason) -def select_jobs_by_partial_id(jobs, partial_id): +def select_jobs_by_partial_id(jobs: typing.List[job.Job], partial_id: str) -> typing.List[job.Job]: selected = [] for j in jobs: if j.plot_id.startswith(partial_id): diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py index 280ba530..37215527 100644 --- a/src/plotman/plot_util.py +++ b/src/plotman/plot_util.py @@ -2,37 +2,40 @@ import os import re import shutil +import typing + from plotman import chiapos +import plotman.job GB = 1_000_000_000 -def df_b(d): +def df_b(d: str) -> int: 'Return free space for directory (in bytes)' usage = shutil.disk_usage(d) return usage.free -def get_k32_plotsize(): +def get_k32_plotsize() -> int: return get_plotsize(32) -def get_plotsize(k): +def get_plotsize(k: int) -> int: return (int)(_get_plotsize_scaler(k) * k * pow(2, k)) -def human_format(num, precision, powerOfTwo=False): +def human_format(num: float, precision: int, powerOfTwo: bool = False) -> str: divisor = 1024 if powerOfTwo else 1000 - + magnitude = 0 while abs(num) >= divisor: magnitude += 1 - num /= divisor + num /= divisor result = (('%.' + str(precision) + 'f%s') % (num, ['', 'K', 'M', 'G', 'T', 'P'][magnitude])) if powerOfTwo and magnitude > 0: result += 'i' - + return result -def time_format(sec): +def time_format(sec: typing.Optional[int]) -> str: if sec is None: return '-' if sec < 60: @@ -40,13 +43,7 @@ def time_format(sec): else: return '%d:%02d' % (int(sec / 3600), int((sec % 3600) / 60)) -def tmpdir_phases_str(tmpdir_phases_pair): - tmpdir = tmpdir_phases_pair[0] - phases = tmpdir_phases_pair[1] - phase_str = ', '.join(['%d:%d' % ph_subph for ph_subph in sorted(phases)]) - return ('%s (%s)' % (tmpdir, phase_str)) - -def split_path_prefix(items): +def split_path_prefix(items: typing.List[str]) -> typing.Tuple[str, typing.List[str]]: if not items: return ('', []) @@ -57,7 +54,7 @@ def split_path_prefix(items): remainders = [ os.path.relpath(i, prefix) for i in items ] return (prefix, remainders) -def list_k32_plots(d): +def list_k32_plots(d: str) -> typing.List[str]: 'List completed k32 plots in a directory (not recursive)' plots = [] for plot in os.listdir(d): @@ -71,22 +68,27 @@ def list_k32_plots(d): return plots -def column_wrap(items, n_cols, filler=None): +def column_wrap( + items: typing.Sequence[object], + n_cols: int, + filler: typing.Optional[str] = None, +) -> typing.List[typing.List[typing.Optional[object]]]: '''Take items, distribute among n_cols columns, and return a set of rows containing the slices of those columns.''' - rows = [] + rows: typing.List[typing.List[typing.Optional[object]]] = [] n_rows = math.ceil(len(items) / n_cols) for row in range(n_rows): row_items = items[row : : n_rows] # Pad and truncate - rows.append( (row_items + ([filler] * n_cols))[:n_cols] ) + padded: typing.List[typing.Optional[object]] = [*row_items, *([filler] * n_cols)] + rows.append(list(padded[:n_cols])) return rows # use k as index to get plotsize_scaler, note that 0 means the value is not calculated yet # we can safely assume that k is never going to be greater than 100, due to the exponential nature of plot file size, this avoids using constants from chiapos _plotsize_scaler_cache = [0.0 for _ in range(0, 101)] -def calc_average_size_of_entry(k, table_index): +def calc_average_size_of_entry(k: int, table_index: int) -> float: ''' calculate the average size of entries in bytes, given k and table_index ''' @@ -94,7 +96,7 @@ def calc_average_size_of_entry(k, table_index): # it is approximately k/8, uses chia's actual park size calculation to get a more accurate estimation return chiapos.CalculateParkSize(k, table_index) / chiapos.kEntriesPerPark -def _get_probability_of_entries_kept(k, table_index): +def _get_probability_of_entries_kept(k: int, table_index: int) -> float: ''' get the probibility of entries in table of table_index that is not dropped ''' @@ -106,11 +108,13 @@ def _get_probability_of_entries_kept(k, table_index): pow_2_k = 2**k if table_index == 5: - return 1 - (1 - 2 / pow_2_k) ** pow_2_k # p5 + # p5 + return 1 - (1 - 2 / pow_2_k) ** pow_2_k # type: ignore[no-any-return] else: - return 1 - (1 - 2 / pow_2_k) ** (_get_probability_of_entries_kept(k, table_index + 1) * pow_2_k) # pt + # pt + return 1 - (1 - 2 / pow_2_k) ** (_get_probability_of_entries_kept(k, table_index + 1) * pow_2_k) # type: ignore[no-any-return] -def _get_plotsize_scaler(k:int): +def _get_plotsize_scaler(k: int) -> float: ''' get scaler for plot size so that the plot size can be calculated by scaler * k * 2 ** k ''' @@ -121,12 +125,12 @@ def _get_plotsize_scaler(k:int): _plotsize_scaler_cache[k] = result return result -def _get_plotsize_scaler_impl(k): +def _get_plotsize_scaler_impl(k: int) -> float: ''' get scaler for plot size so that the plot size can be calculated by scaler * k * 2 ** k ''' - result = 0 + result = 0.0 # there are 7 tables for i in range(1, 8): probability = _get_probability_of_entries_kept(k, i) diff --git a/src/plotman/plotinfo.py b/src/plotman/plotinfo.py index 0f131c16..b7c3a13f 100644 --- a/src/plotman/plotinfo.py +++ b/src/plotman/plotinfo.py @@ -105,8 +105,8 @@ def copy_time_minutes(self) -> int: def copy_time_hours(self) -> float: return self.duration_to_hours(self.copy_time_raw) - def duration_to_minutes(self, duration): + def duration_to_minutes(self, duration: float) -> int: return round(duration / 60) - def duration_to_hours(self, duration): + def duration_to_hours(self, duration: float) -> float: return round(duration / 60 / 60, 2) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 02f4090c..04863491 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -10,6 +10,7 @@ from shutil import copyfile import sys import time +import typing import pendulum @@ -19,14 +20,14 @@ from plotman.job import Job class PlotmanArgParser: - def add_idprefix_arg(self, subparser): + def add_idprefix_arg(self, subparser: argparse.ArgumentParser) -> None: subparser.add_argument( 'idprefix', type=str, nargs='+', help='disambiguating prefix of plot ID') - def parse_args(self): + def parse_args(self) -> typing.Any: parser = argparse.ArgumentParser(description='Chia plotting manager.') sp = parser.add_subparsers(dest='cmd') @@ -90,21 +91,20 @@ def parse_args(self): args = parser.parse_args() return args -def get_term_width(): - columns = 0 +def get_term_width() -> int: try: - (rows, columns) = os.popen('stty size', 'r').read().split() - columns = int(columns) + (rows_string, columns_string) = os.popen('stty size', 'r').read().split() + columns = int(columns_string) except: columns = 120 # 80 is typically too narrow. TODO: make a command line arg. return columns class Iso8601Formatter(logging.Formatter): - def formatTime(self, record, datefmt=None): + def formatTime(self, record: logging.LogRecord, datefmt: typing.Optional[str] = None) -> str: time = pendulum.from_timestamp(timestamp=record.created, tz='local') - return time.isoformat(timespec='microseconds', ) + return time.isoformat(timespec='microseconds') -def main(): +def main() -> None: random.seed() pm_parser = PlotmanArgParser() @@ -123,7 +123,7 @@ def main(): return print(f"No 'plotman.yaml' file exists at expected location: '{config_file_path}'") print(f"To generate a default config file, run: 'plotman config generate'") - return 1 + return if args.config_subcommand == 'generate': if os.path.isfile(config_file_path): overwrite = None @@ -170,6 +170,7 @@ def main(): handler.setFormatter(formatter) root_logger.addHandler(handler) root_logger.setLevel(logging.INFO) + root_logger.info('abc') # # Stay alive, spawning plot jobs @@ -218,7 +219,7 @@ def main(): # Directories report elif args.cmd == 'dirs': - print(reporting.dirs_report(jobs, cfg.directories, cfg.scheduling, get_term_width())) + print(reporting.dirs_report(jobs, cfg.directories, cfg.archiving, cfg.scheduling, get_term_width())) elif args.cmd == 'interactive': interactive.run_interactive( @@ -229,18 +230,21 @@ def main(): # Start running archival elif args.cmd == 'archive': - print('...starting archive loop') - firstit = True - while True: - if not firstit: - print('Sleeping 60s until next iteration...') - time.sleep(60) - jobs = Job.get_running_jobs(cfg.logging.plots) - firstit = False - - archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, cfg.logging, jobs) - for log_message in log_messages: - print(log_message) + if cfg.archiving is None: + print('archiving not configured but is required for this command') + else: + print('...starting archive loop') + firstit = True + while True: + if not firstit: + print('Sleeping 60s until next iteration...') + time.sleep(60) + jobs = Job.get_running_jobs(cfg.logging.plots) + firstit = False + + archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, cfg.logging, jobs) + for log_message in log_messages: + print(log_message) # Debugging: show the destination drive usage schedule diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 670d82ef..20ce8940 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -1,26 +1,27 @@ import math import os +import typing import psutil import texttable as tt # from somewhere? from itertools import groupby -from plotman import archive, job, manager, plot_util +from plotman import archive, configuration, job, manager, plot_util -def abbr_path(path, putative_prefix): +def abbr_path(path: str, putative_prefix: str) -> str: if putative_prefix and path.startswith(putative_prefix): return os.path.relpath(path, putative_prefix) else: return path -def phase_str(phase): +def phase_str(phase: job.Phase) -> str: if not phase.known: return '?:?' return f'{phase.major}:{phase.minor}' -def phases_str(phases, max_num=None): +def phases_str(phases: typing.List[job.Phase], max_num: typing.Optional[int] = None) -> str: '''Take a list of phase-subphase pairs and return them as a compact string''' if not max_num or len(phases) <= max_num: return ' '.join([phase_str(pair) for pair in phases]) @@ -33,10 +34,10 @@ def phases_str(phases, max_num=None): last = ' '.join([phase_str(pair) for pair in phases[n_first + n_elided:]]) return first + elided + last -def n_at_ph(jobs, ph): +def n_at_ph(jobs: typing.List[job.Job], ph: job.Phase) -> int: return sum([1 for j in jobs if j.progress() == ph]) -def n_to_char(n): +def n_to_char(n: int) -> str: n_to_char_map = dict(enumerate(" .:;!")) if n < 0: @@ -46,7 +47,7 @@ def n_to_char(n): return n_to_char_map[n] -def job_viz(jobs): +def job_viz(jobs: typing.List[job.Job]) -> str: # TODO: Rewrite this in a way that ensures we count every job # even if the reported phases don't line up with expectations. result = '' @@ -65,7 +66,7 @@ def job_viz(jobs): # Command: plotman status # Shows a general overview of all running jobs -def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''): +def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optional[int] = None, tmp_prefix: str = '', dst_prefix: str = '') -> str: '''height, if provided, will limit the number of rows in the table, showing first and last rows, row numbers and an elipsis in the middle.''' abbreviate_jobs_list = False @@ -74,7 +75,6 @@ def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''): if height and height < len(jobs) + 1: # One row for header abbreviate_jobs_list = True - if abbreviate_jobs_list: n_rows = height - 2 # Minus one for header, one for ellipsis n_begin_rows = int(n_rows / 2) n_end_rows = n_rows - n_begin_rows @@ -102,7 +102,7 @@ def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''): try: with j.proc.oneshot(): row = [j.plot_id[:8], # Plot ID - j.k, # k size + str(j.k), # k size abbr_path(j.tmpdir, tmp_prefix), # Temp directory abbr_path(j.dstdir, dst_prefix), # Destination directory plot_util.time_format(j.get_time_wall()), # Time wall @@ -127,9 +127,9 @@ def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''): tab.set_max_width(width) tab.set_deco(0) # No borders - return tab.draw() + return tab.draw() # type: ignore[no-any-return] -def summary(jobs, tmp_prefix=''): +def summary(jobs: typing.List[job.Job], tmp_prefix: str = '') -> str: """Creates a small summary of running jobs""" summary = [ @@ -145,7 +145,7 @@ def summary(jobs, tmp_prefix=''): return '\n'.join(summary) -def tmp_dir_report(jobs, dir_cfg, sched_cfg, width, start_row=None, end_row=None, prefix=''): +def tmp_dir_report(jobs: typing.List[job.Job], dir_cfg: configuration.Directories, sched_cfg: configuration.Scheduling, width: int, start_row: typing.Optional[int] = None, end_row: typing.Optional[int] = None, prefix: str = '') -> str: '''start_row, end_row let you split the table up if you want''' tab = tt.Texttable() headings = ['tmp', 'ready', 'phases'] @@ -163,9 +163,9 @@ def tmp_dir_report(jobs, dir_cfg, sched_cfg, width, start_row=None, end_row=None tab.set_max_width(width) tab.set_deco(tt.Texttable.BORDER | tt.Texttable.HEADER ) tab.set_deco(0) # No borders - return tab.draw() + return tab.draw() # type: ignore[no-any-return] -def dst_dir_report(jobs, dstdirs, width, prefix=''): +def dst_dir_report(jobs: typing.List[job.Job], dstdirs: typing.List[str], width: int, prefix: str='') -> str: tab = tt.Texttable() dir2oldphase = manager.dstdirs_to_furthest_phase(jobs) dir2newphase = manager.dstdirs_to_youngest_phase(jobs) @@ -189,9 +189,9 @@ def dst_dir_report(jobs, dstdirs, width, prefix=''): tab.set_max_width(width) tab.set_deco(tt.Texttable.BORDER | tt.Texttable.HEADER ) tab.set_deco(0) # No borders - return tab.draw() + return tab.draw() # type: ignore[no-any-return] -def arch_dir_report(archdir_freebytes, width, prefix=''): +def arch_dir_report(archdir_freebytes: typing.Dict[str, int], width: int, prefix: str = '') -> str: cells = ['%s:%5dG' % (abbr_path(d, prefix), int(int(space) / plot_util.GB)) for (d, space) in sorted(archdir_freebytes.items())] if not cells: @@ -204,10 +204,10 @@ def arch_dir_report(archdir_freebytes, width, prefix=''): tab.add_row(row) tab.set_cols_align('r' * (n_columns)) tab.set_deco(tt.Texttable.VLINES) - return tab.draw() + return tab.draw() # type: ignore[no-any-return] # TODO: remove this -def dirs_report(jobs, dir_cfg, arch_cfg, sched_cfg, width): +def dirs_report(jobs: typing.List[job.Job], dir_cfg: configuration.Directories, arch_cfg: typing.Optional[configuration.Archiving], sched_cfg: configuration.Scheduling, width: int) -> str: dst_dir = dir_cfg.get_dst_directories() reports = [ tmp_dir_report(jobs, dir_cfg, sched_cfg, width), diff --git a/tox.ini b/tox.ini index 74aec0d5..ab964c07 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,9 @@ extras = commands = pytest --capture=no --verbose --cov=plotman --cov-report=term-missing --cov-report=xml:{toxinidir}/coverage.xml --pyargs plotman -[testenv:check] +[testenv:check-manifest] extras = - test + checks commands = check-manifest --verbose {toxinidir} @@ -27,3 +27,9 @@ commands = coverage xml -o coverage.xml coverage report --fail-under=35 --ignore-errors --show-missing diff-cover --fail-under=100 {posargs:--compare-branch=development} coverage.xml + +[testenv:check-hints] +extras = + checks +commands = + mypy --package plotman From d0a36833d65010b42e30624e30fa63f2130dd735 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 12 Jun 2021 20:47:17 -0400 Subject: [PATCH 013/109] include mypy.ini in MANIEFST.in --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 5510584d..08d64bbe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include LICENSE* include README.md include *.md include VERSION +include mypy.ini include tox.ini include .coveragerc recursive-include src *.py From 1efa5a91f2e19f5007a9ff44db5a45a3fe53f825 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 12 Jun 2021 21:04:06 -0400 Subject: [PATCH 014/109] fixup tox check-hints-py* --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ab964c07..f75a3378 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ commands = coverage report --fail-under=35 --ignore-errors --show-missing diff-cover --fail-under=100 {posargs:--compare-branch=development} coverage.xml -[testenv:check-hints] +[testenv:check-hints-py{37,38,39}] extras = checks commands = From 2a6727ca64446fbd8fbb45a00e3ef082699114d9 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 12 Jun 2021 22:15:21 -0400 Subject: [PATCH 015/109] use typing_extensions.Protocol --- setup.cfg | 1 + src/plotman/chia.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 551397ef..021a3500 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,7 @@ install_requires = psutil ~= 5.8 pyyaml ~= 5.4 texttable ~= 1.6 + typing-extensions ~= 3.10 [options.packages.find] where=src diff --git a/src/plotman/chia.py b/src/plotman/chia.py index 27cbddc2..7a2c7d0c 100644 --- a/src/plotman/chia.py +++ b/src/plotman/chia.py @@ -5,10 +5,11 @@ import click from pathlib import Path +import typing_extensions -class CommandProtocol(typing.Protocol): - def make_context(self, info_name:str, args:typing.List[str]) -> click.Context: +class CommandProtocol(typing_extensions.Protocol): + def make_context(self, info_name: str, args: typing.List[str]) -> click.Context: ... def __call__(self) -> None: From ac623a57b34fa591dddc02372491398d4e3de3e2 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 12 Jun 2021 23:07:18 -0400 Subject: [PATCH 016/109] fixup tox.ini for mypy --- mypy.ini | 1 + setup.cfg | 1 + tox.ini | 1 + 3 files changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 71319f45..2b2c6b0f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,7 @@ [mypy] show_error_codes = true strict = true +mypy_path = src/ [mypy-appdirs] ignore_missing_imports = true diff --git a/setup.cfg b/setup.cfg index 021a3500..f922b21e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,6 +70,7 @@ checks = check-manifest ~= 0.46 mypy == 0.902 types-pkg_resources ~= 0.1.2 + %(test)s [options.data_files] config = src/plotman/resources/plotman.yaml diff --git a/tox.ini b/tox.ini index f75a3378..77170cb1 100644 --- a/tox.ini +++ b/tox.ini @@ -29,6 +29,7 @@ commands = diff-cover --fail-under=100 {posargs:--compare-branch=development} coverage.xml [testenv:check-hints-py{37,38,39}] +changedir = {toxinidir} extras = checks commands = From c73c63d813e9cf6c9e36bc541ba6d3e5cf023189 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Sun, 13 Jun 2021 15:39:41 +0000 Subject: [PATCH 017/109] Add docker build --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ca379a55..609e8dfd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ __pycache__ venv .DS_Store .vscode +*.code-workspace src/plotman.egg-info From 834f2fe3514b195eccdb345b9e4c8793e67d1520 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Sun, 13 Jun 2021 15:39:59 +0000 Subject: [PATCH 018/109] Add docker build --- .dockerfile | 4 +++ Dockerfile | 60 +++++++++++++++++++++++++++++++++++++++++ build-docker-plotman.sh | 20 ++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 .dockerfile create mode 100644 Dockerfile create mode 100755 build-docker-plotman.sh diff --git a/.dockerfile b/.dockerfile new file mode 100644 index 00000000..756b8274 --- /dev/null +++ b/.dockerfile @@ -0,0 +1,4 @@ +.git +.github +.coveragerc +.gitignore diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4435ec98 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +# Build deployable artifacts +FROM ubuntu:latest as plotman-builder + +ARG CHIA_BRANCH + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 ansible tar bash ca-certificates git openssl unzip wget python3-pip sudo acl build-essential python3-dev python3.8-venv python3.8-distutils apt nfs-common python-is-python3 cmake libsodium-dev g++ + +RUN echo "cloning ${CHIA_BRANCH}" +RUN git clone --branch ${CHIA_BRANCH} https://github.com/Chia-Network/chia-blockchain.git \ +&& cd chia-blockchain \ +&& git submodule update --init mozilla-ca \ +&& chmod +x install.sh + +WORKDIR /chia-blockchain +# Placeholder for patches +RUN /usr/bin/sh ./install.sh + +COPY . /plotman + +RUN . ./activate \ +&& pip install -e /plotman \ +&& deactivate + +# Build deployment container +FROM ubuntu:latest as plotman + +ARG UID=10001 +ARG GID=10001 + +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ +&& DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 python3.8-venv python3.8-distutils ca-certificates tzdata vim ssh less rsync git tmux libsodium23 \ +&& apt-get clean all \ +&& rm -rf /var/lib/apt/lists + +COPY --from=plotman-builder /chia-blockchain /chia-blockchain +COPY --from=plotman-builder /plotman /plotman + +RUN groupadd -g ${GID} chia +RUN useradd -m -u ${UID} -g ${GID} chia + +RUN mkdir -p /data/chia/tmp \ +&& mkdir -p /data/chia/plots \ +&& mkdir -p /data/chia/logs + +VOLUME ["/data/chia/tmp","/data/chia/plots","/data/chia/logs"] + +RUN chown -R chia:chia /chia-blockchain \ +&& chown -R chia:chia /plotman \ +&& chown -R chia:chia /data/chia + +WORKDIR /chia-blockchain +USER chia + +ENV VIRTUAL_ENV="/chia-blockchain/venv" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Kick off plots (assumes the environemnt is good to go) +CMD ["/bin/bash", "-c", "plotman plot" ] +# Alternative command to simply provide shell environment +#CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ] diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh new file mode 100755 index 00000000..7e9a3be7 --- /dev/null +++ b/build-docker-plotman.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +DOCKER_REGISTRY="graemes" +LOCAL_REGISTRY="registry.graemes.com/graemes" +PROJECT="chia-plotman" +TAG="plotter" +CHIA_BRANCH="1.1.7" + +docker rmi ${LOCAL_REGISTRY}/${PROJECT}:${TAG} + +docker build . \ + --squash \ + --build-arg CHIA_BRANCH=${CHIA_BRANCH} \ + -f Dockerfile \ + -t ${LOCAL_REGISTRY}/${PROJECT}:${TAG} + +# -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} \ + +docker push ${LOCAL_REGISTRY}/${PROJECT}:${TAG} +#docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG} From 32f33af7fad34ca4bff65734050f4aa29c3e4646 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 13 Jun 2021 11:57:50 -0400 Subject: [PATCH 019/109] use attr.ib instead of desert.ib to satisfy mypy --- src/plotman/configuration.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 33f8589e..a74acd89 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -92,12 +92,15 @@ class ArchivingTarget: transfer_process_name: str transfer_process_argument_prefix: str # TODO: mutable attribute... - env: Dict[str, Optional[str]] = desert.ib( + # TODO: should be a desert.ib() but mypy doesn't understand it then + env: Dict[str, Optional[str]] = attr.ib( factory=dict, - marshmallow_field=marshmallow.fields.Dict( - keys=marshmallow.fields.String(), - values=CustomStringField(allow_none=True), - ), + metadata={ + 'marshmallow_field': marshmallow.fields.Dict( + keys=marshmallow.fields.String(), + values=CustomStringField(allow_none=True), + ) + }, ) disk_space_path: Optional[str] = None disk_space_script: Optional[str] = None @@ -113,12 +116,15 @@ class PresetTargetDefinitions: class Archiving: target: str # TODO: mutable attribute... - env: Dict[str, str] = desert.ib( + # TODO: should be a desert.ib() but mypy doesn't understand it then + env: Dict[str, str] = attr.ib( factory=dict, - marshmallow_field=marshmallow.fields.Dict( - keys=marshmallow.fields.String(), - values=CustomStringField(), - ), + metadata={ + 'marshmallow_field': marshmallow.fields.Dict( + keys=marshmallow.fields.String(), + values=CustomStringField(), + ), + }, ) index: int = 0 # If not explicit, "index" will default to 0 target_definitions: Dict[str, ArchivingTarget] = attr.ib(factory=dict) From 7d935dc75a21ac3cf8086117738435f1c6507906 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 13 Jun 2021 13:11:10 -0400 Subject: [PATCH 020/109] use desert._make._DESERT_SENTINEL --- src/plotman/configuration.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index a74acd89..3181b5a9 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -9,6 +9,8 @@ import appdirs import attr import desert +# TODO: should be a desert.ib() but mypy doesn't understand it then, see below +import desert._make import marshmallow import pendulum import yaml @@ -96,10 +98,12 @@ class ArchivingTarget: env: Dict[str, Optional[str]] = attr.ib( factory=dict, metadata={ - 'marshmallow_field': marshmallow.fields.Dict( - keys=marshmallow.fields.String(), - values=CustomStringField(allow_none=True), - ) + desert._make._DESERT_SENTINEL: { + 'marshmallow_field': marshmallow.fields.Dict( + keys=marshmallow.fields.String(), + values=CustomStringField(allow_none=True), + ) + }, }, ) disk_space_path: Optional[str] = None @@ -120,10 +124,12 @@ class Archiving: env: Dict[str, str] = attr.ib( factory=dict, metadata={ - 'marshmallow_field': marshmallow.fields.Dict( - keys=marshmallow.fields.String(), - values=CustomStringField(), - ), + desert._make._DESERT_SENTINEL: { + 'marshmallow_field': marshmallow.fields.Dict( + keys=marshmallow.fields.String(), + values=CustomStringField(), + ) + }, }, ) index: int = 0 # If not explicit, "index" will default to 0 From c453d38ba9f43bd6e091b85ec4b08c801a0db75d Mon Sep 17 00:00:00 2001 From: tcinbis Date: Tue, 18 May 2021 08:44:59 +0200 Subject: [PATCH 021/109] Adding json flag to status command This allows for printing the status report in a more machine readable json format. --- src/plotman/job.py | 21 +++++++++++++++++++++ src/plotman/plotman.py | 18 ++++++++++++------ src/plotman/reporting.py | 11 ++++++++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 6f19c755..66377917 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -19,6 +19,7 @@ import psutil from plotman import chia +from plotman.reporting import phase_str def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: @@ -407,6 +408,26 @@ def status_str_long(self) -> str: dst = self.dstdir, logfile = self.logfile ) + + def to_dict(self): + '''Exports important information as dictionary.''' + # TODO: Check if being called in oneshot context to improve performance + return dict( + plot_id=self.plot_id[:8], + k=self.k, + tmp_dir=self.tmpdir, + dst_dir=self.dstdir, + progress=phase_str(self.progress()), + tmp_usage=self.get_tmp_usage(), + pid=self.proc.pid, + run_status=self.get_run_status(), + mem_usage=self.get_mem_usage(), + time_wall=self.get_time_wall(), + time_user=self.get_time_user(), + time_sys=self.get_time_sys(), + time_iowait=self.get_time_iowait() + ) + def get_mem_usage(self) -> int: # Total, inc swapped diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 04863491..a844905a 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -33,7 +33,9 @@ def parse_args(self) -> typing.Any: sp.add_parser('version', help='print the version') - sp.add_parser('status', help='show current plotting status') + p_status = sp.add_parser('status', help='show current plotting status') + p_status.add_argument("--json", action="store_true", + help="export status report in json format") sp.add_parser('dirs', help='show directories info') @@ -210,11 +212,15 @@ def main() -> None: # Status report if args.cmd == 'status': - result = "{0}\n\n{1}\n\nUpdated at: {2}".format( - reporting.status_report(jobs, get_term_width()), - reporting.summary(jobs), - datetime.datetime.today().strftime("%c"), - ) + if args.json: + # convert jobs list into json + result = reporting.json_report(jobs) + else: + result = "{0}\n\n{1}\n\nUpdated at: {2}".format( + reporting.status_report(jobs, get_term_width()), + reporting.summary(jobs), + datetime.datetime.today().strftime("%c"), + ) print(result) # Directories report diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 20ce8940..3cd5cb53 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -1,3 +1,4 @@ +import json import math import os import typing @@ -5,7 +6,7 @@ import psutil import texttable as tt # from somewhere? from itertools import groupby - +from collections import defaultdict from plotman import archive, configuration, job, manager, plot_util @@ -222,3 +223,11 @@ def dirs_report(jobs: typing.List[job.Job], dir_cfg: configuration.Directories, ]) return '\n'.join(reports) + '\n' + +def json_report(jobs): + jobs_dict = defaultdict(list) + for _, j in enumerate(sorted(jobs, key=job.Job.get_time_wall)): + with j.proc.oneshot(): + jobs_dict["jobs"].append(j.to_dict()) + return json.dumps(jobs_dict) + From 808666afdfc6ae3f80a23b6d78cfc8bdab9b2135 Mon Sep 17 00:00:00 2001 From: tcinbis Date: Tue, 18 May 2021 09:11:56 +0200 Subject: [PATCH 022/109] Adding latest status fields to json status --- src/plotman/reporting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 3cd5cb53..2f4c7655 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -1,3 +1,4 @@ +import time import json import math import os @@ -229,5 +230,7 @@ def json_report(jobs): for _, j in enumerate(sorted(jobs, key=job.Job.get_time_wall)): with j.proc.oneshot(): jobs_dict["jobs"].append(j.to_dict()) + jobs_dict["total_jobs"] = len(jobs) + jobs_dict["updated"] = time.time() return json.dumps(jobs_dict) From 0d6412b4ee194f9d4cda3df139837ca91bbd196d Mon Sep 17 00:00:00 2001 From: tcinbis Date: Mon, 14 Jun 2021 10:08:34 +0200 Subject: [PATCH 023/109] Moving phase_str into Phase class as __str__ --- src/plotman/job.py | 9 ++++++--- src/plotman/reporting.py | 14 ++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 66377917..9d987095 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -19,8 +19,6 @@ import psutil from plotman import chia -from plotman.reporting import phase_str - def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: '''Return phase 2-tuples for jobs running on tmpdir d''' @@ -133,6 +131,11 @@ def list_from_tuples( ) -> typing.List["Phase"]: return [cls.from_tuple(t) for t in l] + def __str__(self): + if not self.known: + return '?:?' + return f'{self.major}:{self.minor}' + # TODO: be more principled and explicit about what we cache vs. what we look up # dynamically from the logfile class Job: @@ -417,7 +420,7 @@ def to_dict(self): k=self.k, tmp_dir=self.tmpdir, dst_dir=self.dstdir, - progress=phase_str(self.progress()), + progress=str(self.progress()), tmp_usage=self.get_tmp_usage(), pid=self.proc.pid, run_status=self.get_run_status(), diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 2f4c7655..b668d7ff 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -17,23 +17,17 @@ def abbr_path(path: str, putative_prefix: str) -> str: else: return path -def phase_str(phase: job.Phase) -> str: - if not phase.known: - return '?:?' - - return f'{phase.major}:{phase.minor}' - def phases_str(phases: typing.List[job.Phase], max_num: typing.Optional[int] = None) -> str: '''Take a list of phase-subphase pairs and return them as a compact string''' if not max_num or len(phases) <= max_num: - return ' '.join([phase_str(pair) for pair in phases]) + return ' '.join([str(pair) for pair in phases]) else: n_first = math.floor(max_num / 2) n_last = max_num - n_first n_elided = len(phases) - (n_first + n_last) - first = ' '.join([phase_str(pair) for pair in phases[:n_first]]) + first = ' '.join([str(pair) for pair in phases[:n_first]]) elided = " [+%d] " % n_elided - last = ' '.join([phase_str(pair) for pair in phases[n_first + n_elided:]]) + last = ' '.join([str(pair) for pair in phases[n_first + n_elided:]]) return first + elided + last def n_at_ph(jobs: typing.List[job.Job], ph: job.Phase) -> int: @@ -108,7 +102,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona abbr_path(j.tmpdir, tmp_prefix), # Temp directory abbr_path(j.dstdir, dst_prefix), # Destination directory plot_util.time_format(j.get_time_wall()), # Time wall - phase_str(j.progress()), # Overall progress (major:minor) + str(j.progress()), # Overall progress (major:minor) plot_util.human_format(j.get_tmp_usage(), 0), # Current temp file size j.proc.pid, # System pid j.get_run_status(), # OS status for the job process From 3fe5d820048d13922eeda76c1e92fa9de39ce9fb Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 14 Jun 2021 20:26:01 -0400 Subject: [PATCH 024/109] add missing plotman export changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c82dcad..5979d90a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Added +- `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) + ## [0.4.1] - 2021-06-11 ### Fixed - Archival disk space check finds drives with multiple mount points again. From 873f8b0a8b6a9d68d1cd6c6bfaa5e6ad6271907b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 14 Jun 2021 20:56:40 -0400 Subject: [PATCH 025/109] changelog, type hints, and tweaks --- CHANGELOG.md | 1 + src/plotman/job.py | 6 +++--- src/plotman/reporting.py | 20 ++++++++++++-------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5979d90a..4fb4b02a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Added - `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) +- `--json` option for `plotman status`. ([#549](https://github.com/ericaltendorf/plotman/pull/549)) ## [0.4.1] - 2021-06-11 ### Fixed diff --git a/src/plotman/job.py b/src/plotman/job.py index 9d987095..b5c4a838 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -20,6 +20,7 @@ from plotman import chia + def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: '''Return phase 2-tuples for jobs running on tmpdir d''' return sorted([j.progress() for j in all_jobs if j.tmpdir == d]) @@ -131,7 +132,7 @@ def list_from_tuples( ) -> typing.List["Phase"]: return [cls.from_tuple(t) for t in l] - def __str__(self): + def __str__(self) -> str: if not self.known: return '?:?' return f'{self.major}:{self.minor}' @@ -412,9 +413,8 @@ def status_str_long(self) -> str: logfile = self.logfile ) - def to_dict(self): + def to_dict(self) -> typing.Dict[str, object]: '''Exports important information as dictionary.''' - # TODO: Check if being called in oneshot context to improve performance return dict( plot_id=self.plot_id[:8], k=self.k, diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index b668d7ff..3e6e0bfd 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -7,7 +7,6 @@ import psutil import texttable as tt # from somewhere? from itertools import groupby -from collections import defaultdict from plotman import archive, configuration, job, manager, plot_util @@ -219,12 +218,17 @@ def dirs_report(jobs: typing.List[job.Job], dir_cfg: configuration.Directories, return '\n'.join(reports) + '\n' -def json_report(jobs): - jobs_dict = defaultdict(list) - for _, j in enumerate(sorted(jobs, key=job.Job.get_time_wall)): +def json_report(jobs: typing.List[job.Job]) -> str: + jobs_dicts = [] + for j in sorted(jobs, key=job.Job.get_time_wall): with j.proc.oneshot(): - jobs_dict["jobs"].append(j.to_dict()) - jobs_dict["total_jobs"] = len(jobs) - jobs_dict["updated"] = time.time() - return json.dumps(jobs_dict) + jobs_dicts.append(j.to_dict()) + + stuff = { + "jobs": jobs_dicts, + "total_jobs": len(jobs), + "updated": time.time(), + } + + return json.dumps(stuff) From 42d43d85863e94ac1a0e50e16e84e63bc4bc3e02 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 14 Jun 2021 21:13:48 -0400 Subject: [PATCH 026/109] add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fb4b02a..09570c70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) - `--json` option for `plotman status`. ([#549](https://github.com/ericaltendorf/plotman/pull/549)) +- If the tmp drive selected for a plot is also listed as a dst drive then plotman will use the same drive for both. ([#643](https://github.com/ericaltendorf/plotman/pull/643)) ## [0.4.1] - 2021-06-11 ### Fixed From 9068a47f72ef20db404a0c292e4b62040cc177d0 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 14 Jun 2021 21:53:29 -0400 Subject: [PATCH 027/109] reformat changelog --- CHANGELOG.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09570c70..0b291813 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,18 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Added -- `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) -- `--json` option for `plotman status`. ([#549](https://github.com/ericaltendorf/plotman/pull/549)) -- If the tmp drive selected for a plot is also listed as a dst drive then plotman will use the same drive for both. ([#643](https://github.com/ericaltendorf/plotman/pull/643)) +- `plotman export` command to output summaries from plot logs in `.csv` format. + ([#557](https://github.com/ericaltendorf/plotman/pull/557)) +- `--json` option for `plotman status`. + ([#549](https://github.com/ericaltendorf/plotman/pull/549)) +- If the tmp drive selected for a plot is also listed as a dst drive then plotman will use the same drive for both. + ([#643](https://github.com/ericaltendorf/plotman/pull/643)) ## [0.4.1] - 2021-06-11 ### Fixed - Archival disk space check finds drives with multiple mount points again. This fixes a regression introduced in v0.4.1. - [#773](https://github.com/ericaltendorf/plotman/issues/773) + ([#773](https://github.com/ericaltendorf/plotman/issues/773)) - `plotman dirs` does not fail for every invocation. `TypeError: dirs_report() missing 1 required positional argument: 'width'` - [#778](https://github.com/ericaltendorf/plotman/issues/778) + ([#778](https://github.com/ericaltendorf/plotman/issues/778)) ## [0.4] - 2021-06-10 ### Fixed From de6ea5e95b9f12f518bc98ed44e9c84f0e36301f Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Tue, 15 Jun 2021 23:43:03 +0100 Subject: [PATCH 028/109] Moved tmp_overrides from directories into scheduling, added extra configuration options with description and examples --- src/plotman/resources/plotman.yaml | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 9835d16e..f50016d2 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -118,26 +118,33 @@ scheduling: # How often the daemon wakes to consider starting a new plot job, in seconds. polling_time_s: 20 - # Optional: Allows overriding some scheduling characteristics of certain tmp - # directories. This contains a map of tmp directory names to - # attributes. If a tmp directory and attribute is not listed here, - # it uses the default attribute setting from the main configuration. + # Optional: Allows the overriding of some scheduling characteristics of the + # tmp directories specified here. + # This contains a map of tmp directory names to attributes. If a tmp directory + # and attribute is not listed here, the default attribute setting from the main + # configuration will be used # # Currently support override parameters: - # - tmpdir_stagger_phase_major - # - tmpdir_stagger_phase_minor + # - tmpdir_stagger_phase_major (requires tmpdir_stagger_phase_minor) + # - tmpdir_stagger_phase_minor (requires tmpdir_stagger_phase_major) # - tmpdir_stagger_phase_limit # - tmpdir_max_jobs tmp_overrides: - # In this example, /mnt/tmp/00 is larger than the other tmp - # dirs and it can hold more plots than the default, allowing - # more simultaneous plots and they are being started earlier - # than the global setting above. + # In this example, /mnt/tmp/00 is larger and faster than the + # other tmp dirs and it can hold more plots than the default, + # allowing more simultaneous plots, so they are being started + # earlier than the global setting above. "/mnt/tmp/00": tmpdir_stagger_phase_major: 1 tmpdir_stagger_phase_minor: 5 - tmpdir_stagger_phase_limit: 1 tmpdir_max_jobs: 5 + # Here, /mnt/tmp/03 is smaller, so a different config might be + # to space the phase stagger further apart and only allow 2 jobs + # to run concurrently in it + @/mnt/tmp/03:: + tmpdir_stagger_phase_major: 3 + tmpdir_stagger_phase_minor: 1 + tmpdir_max_jobs: 2 # Plotting parameters. These are pass-through parameters to chia plots create. # See documentation at From 38d7a743f0f43f66e396321254d06b2ac67c0900 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 00:24:30 +0100 Subject: [PATCH 029/109] additional overrides in phases_permit_new_job and different wait_reason messages --- src/plotman/manager.py | 56 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 425f9aa1..ccdf8d37 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -53,38 +53,38 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo if len(phases) == 0: return True + # Assign variables + major = sched_cfg.tmpdir_stagger_phase_major + minor = sched_cfg.tmpdir_stagger_phase_minor + # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py + stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit + + # Limit the total number of jobs per tmp dir. Default to overall max + # jobs configuration, but restrict to any configured overrides. + max_plots = sched_cfg.tmpdir_max_jobs + # Check if any overrides exist for the current job if sched_cfg.tmp_overrides is not None and d in sched_cfg.tmp_overrides: curr_overrides = sched_cfg.tmp_overrides[d] - # Check and apply overrides for major, minor and phase limit - if curr_overrides.tmpdir_stagger_phase_major is not None: - major = curr_overrides.tmpdir_stagger_phase_major - else: - major = sched_cfg.tmpdir_stagger_phase_major - if curr_overrides.tmpdir_stagger_phase_minor is not None: - minor = curr_overrides.tmpdir_stagger_phase_minor - else: - minor = sched_cfg.tmpdir_stagger_phase_minor + # Check for and assign major & minor phase overrides + if curr_overrides.tmpdir_stagger_phase_major is not None: + major = curr_overrides.tmpdir_stagger_phase_major + minor = curr_overrides.tmpdir_stagger_phase_minor + # Check for and assign stagger phase limit override + if curr_overrides.tmpdir_stagger_phase_limit is not None: + stagger_phase_limit = curr_overrides.tmpdir_stagger_phase_limit + # Check for and assign stagger phase limit override + if curr_overrides.tmpdir_max_jobs is not None: + max_plots = curr_overrides.tmpdir_max_jobs + milestone = job.Phase(major,minor,) - if curr_overrides.tmpdir_stagger_phase_limit is not None: - stagger_phase_limit = curr_overrides.tmpdir_stagger_phase_limit - else: - stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit - # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py - if len([p for p in phases if p < milestone]) >= stagger_phase_limit: + + # Check if phases pass the criteria + if len([p for p in phases if p < milestone]) >= stagger_phase_limit or len(phases) >= max_plots: return False - - # Limit the total number of jobs per tmp dir. Default to the overall max - # jobs configuration, but restrict to any configured overrides. - if curr_overrides.tmpdir_max_jobs is not None: - max_plots = curr_overrides.tmpdir_max_jobs else: - max_plots = sched_cfg.tmpdir_max_jobs - if len(phases) >= max_plots: - return False - - return True + return True def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: plotman.configuration.Scheduling, plotting_cfg: plotman.configuration.Plotting, log_cfg: plotman.configuration.Logging) -> typing.Tuple[bool, str]: jobs = job.Job.get_running_jobs(log_cfg.plots) @@ -94,9 +94,9 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: youngest_job_age = min(jobs, key=job.Job.get_time_wall).get_time_wall() if jobs else MAX_AGE global_stagger = int(sched_cfg.global_stagger_m * MIN) if (youngest_job_age < global_stagger): - wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger) + wait_reason = 'global stagger (%ds/%ds)' % (youngest_job_age, global_stagger) elif len(jobs) >= sched_cfg.global_max_jobs: - wait_reason = 'max jobs (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger) + wait_reason = 'global max jobs reached (%d)' % (sched_cfg.global_max_jobs) else: tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp] eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases @@ -105,7 +105,7 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: for (d, phases) in eligible ] if not eligible: - wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger) + wait_reason = 'waiting for phase match' else: # Plot to oldest tmpdir. tmpdir = max(rankable, key=operator.itemgetter(1))[0] From cc02768bb10720414f010ffc5771a2488299b6d1 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 02:17:56 +0100 Subject: [PATCH 030/109] Moved overrides to scheduling and added extras --- src/plotman/configuration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 96c3365b..493f38a0 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -209,9 +209,11 @@ def maybe_create_scripts(self, temp: str) -> None: @attr.frozen class TmpOverrides: tmpdir_stagger_phase_major: Optional[int] = None - tmpdir_stagger_phase_minor: Optional[int] = None + tmpdir_stagger_phase_minor: Optional[int] = None tmpdir_stagger_phase_limit: Optional[int] = None tmpdir_max_jobs: Optional[int] = None + + raise Exception(tmpdir_stagger_phase_major) @attr.frozen class Logging: From 5720fc0f1fd3ada84d1514278068f1cd9c9c42da Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 03:28:08 +0100 Subject: [PATCH 031/109] Update configuration.py removed raise Exception as no longer testing --- src/plotman/configuration.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 493f38a0..3608871a 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -212,8 +212,6 @@ class TmpOverrides: tmpdir_stagger_phase_minor: Optional[int] = None tmpdir_stagger_phase_limit: Optional[int] = None tmpdir_max_jobs: Optional[int] = None - - raise Exception(tmpdir_stagger_phase_major) @attr.frozen class Logging: From 1ec475c0c000bb4e6c11cd58420e0ef830a9c815 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 03:37:23 +0100 Subject: [PATCH 032/109] Updated for #758 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b291813..744c5845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Added +- optional tmpdir_overrides added for each specified tmpdir +- ([#758](https://github.com/ericaltendorf/plotman/pull/758)) - `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) - `--json` option for `plotman status`. From 5b000b188a7a5d0de93dcea473beb713c0f402dd Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 04:06:44 +0100 Subject: [PATCH 033/109] Move tmpdir_overrides from Directories to Scheduling --- src/plotman/_tests/manager_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py index 0c0b752e..a2ecd381 100755 --- a/src/plotman/_tests/manager_test.py +++ b/src/plotman/_tests/manager_test.py @@ -15,15 +15,15 @@ def sched_cfg() -> configuration.Scheduling: polling_time_s=2, tmpdir_stagger_phase_major=3, tmpdir_stagger_phase_minor=0, - tmpdir_max_jobs=3 + tmpdir_max_jobs=3, + tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4) ) @pytest.fixture def dir_cfg() -> configuration.Directories: return configuration.Directories( tmp=["/var/tmp", "/tmp"], - dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"], - tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)} + dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"]} ) def test_permit_new_job_post_milestone(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None: From b691b0415c8b16553629ca81d7a0253320fa7652 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 04:12:22 +0100 Subject: [PATCH 034/109] Update manager_test.py Ad missing } --- src/plotman/_tests/manager_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py index a2ecd381..006b6539 100755 --- a/src/plotman/_tests/manager_test.py +++ b/src/plotman/_tests/manager_test.py @@ -16,7 +16,7 @@ def sched_cfg() -> configuration.Scheduling: tmpdir_stagger_phase_major=3, tmpdir_stagger_phase_minor=0, tmpdir_max_jobs=3, - tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4) + tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)} ) @pytest.fixture From dcd419f6698af7fe2ba485bbb6028441c9841f37 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 04:18:47 +0100 Subject: [PATCH 035/109] Update manager_test.py Correcting a stupid mistake --- src/plotman/_tests/manager_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py index 006b6539..d9fbae85 100755 --- a/src/plotman/_tests/manager_test.py +++ b/src/plotman/_tests/manager_test.py @@ -23,7 +23,7 @@ def sched_cfg() -> configuration.Scheduling: def dir_cfg() -> configuration.Directories: return configuration.Directories( tmp=["/var/tmp", "/tmp"], - dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"]} + dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"] ) def test_permit_new_job_post_milestone(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None: From 44fd610caacba7615280762ab105e8e49ff27e50 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 04:23:14 +0100 Subject: [PATCH 036/109] Correcting typos --- src/plotman/resources/plotman.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index f50016d2..a88dc549 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -141,7 +141,7 @@ scheduling: # Here, /mnt/tmp/03 is smaller, so a different config might be # to space the phase stagger further apart and only allow 2 jobs # to run concurrently in it - @/mnt/tmp/03:: + "/mnt/tmp/03": tmpdir_stagger_phase_major: 3 tmpdir_stagger_phase_minor: 1 tmpdir_max_jobs: 2 From a8599fab0324d969f42c25bd94828a49910f75e2 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Wed, 16 Jun 2021 04:32:57 +0100 Subject: [PATCH 037/109] remove unexpected whitespaces --- src/plotman/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 3608871a..96c3365b 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -209,7 +209,7 @@ def maybe_create_scripts(self, temp: str) -> None: @attr.frozen class TmpOverrides: tmpdir_stagger_phase_major: Optional[int] = None - tmpdir_stagger_phase_minor: Optional[int] = None + tmpdir_stagger_phase_minor: Optional[int] = None tmpdir_stagger_phase_limit: Optional[int] = None tmpdir_max_jobs: Optional[int] = None From 9299f28cdd905161f1e11dadacf960000d89d075 Mon Sep 17 00:00:00 2001 From: racergoodwin Date: Wed, 16 Jun 2021 11:21:42 +0100 Subject: [PATCH 038/109] added global_ to variables defined in scheduling --- src/plotman/configuration.py | 8 ++++---- src/plotman/manager.py | 8 ++++---- src/plotman/resources/plotman.yaml | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 96c3365b..09e8ecc0 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -270,10 +270,10 @@ class Scheduling: global_max_jobs: int global_stagger_m: int polling_time_s: int - tmpdir_max_jobs: int - tmpdir_stagger_phase_major: int - tmpdir_stagger_phase_minor: int - tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 + global_tmpdir_max_jobs: int + global_tmpdir_stagger_phase_major: int + global_tmpdir_stagger_phase_minor: int + global_tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 tmp_overrides: Optional[Dict[str, TmpOverrides]] = None @attr.frozen diff --git a/src/plotman/manager.py b/src/plotman/manager.py index ccdf8d37..af1456c4 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -54,14 +54,14 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo return True # Assign variables - major = sched_cfg.tmpdir_stagger_phase_major - minor = sched_cfg.tmpdir_stagger_phase_minor + major = sched_cfg.global_tmpdir_stagger_phase_major + minor = sched_cfg.global_tmpdir_stagger_phase_minor # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py - stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit + stagger_phase_limit = sched_cfg.global_tmpdir_stagger_phase_limit # Limit the total number of jobs per tmp dir. Default to overall max # jobs configuration, but restrict to any configured overrides. - max_plots = sched_cfg.tmpdir_max_jobs + max_plots = sched_cfg.global_tmpdir_max_jobs # Check if any overrides exist for the current job if sched_cfg.tmp_overrides is not None and d in sched_cfg.tmp_overrides: diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index a88dc549..6212ecb2 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -101,13 +101,13 @@ scheduling: # e.g, with default settings, a new plot will start only when your plot # reaches phase [2 : 1] on your temp drive. This setting takes precidence # over global_stagger_m - tmpdir_stagger_phase_major: 2 - tmpdir_stagger_phase_minor: 1 + global_tmpdir_stagger_phase_major: 2 + global_tmpdir_stagger_phase_minor: 1 # Optional: default is 1 - tmpdir_stagger_phase_limit: 1 + global_tmpdir_stagger_phase_limit: 1 # Don't run more than this many jobs at a time on a single temp dir. - tmpdir_max_jobs: 3 + global_tmpdir_max_jobs: 3 # Don't run more than this many jobs at a time in total. global_max_jobs: 12 From 9b448e8a3ff21740243db170d9e6921fab4861e5 Mon Sep 17 00:00:00 2001 From: racergoodwin Date: Wed, 16 Jun 2021 11:35:23 +0100 Subject: [PATCH 039/109] update _test/manager_test.py --- src/plotman/_tests/manager_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py index d9fbae85..98710087 100755 --- a/src/plotman/_tests/manager_test.py +++ b/src/plotman/_tests/manager_test.py @@ -13,9 +13,9 @@ def sched_cfg() -> configuration.Scheduling: global_max_jobs=1, global_stagger_m=2, polling_time_s=2, - tmpdir_stagger_phase_major=3, - tmpdir_stagger_phase_minor=0, - tmpdir_max_jobs=3, + global_tmpdir_stagger_phase_major=3, + global_tmpdir_stagger_phase_minor=0, + global_tmpdir_max_jobs=3, tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)} ) From 85d95d6893dacf625c65a21b4717c8a7a786c614 Mon Sep 17 00:00:00 2001 From: racergoodwin Date: Wed, 16 Jun 2021 11:50:24 +0100 Subject: [PATCH 040/109] added seperate check for tmpdir_stagger_phase_minor and removed global_ from tmpdir variables --- src/plotman/_tests/manager_test.py | 6 +++--- src/plotman/configuration.py | 8 ++++---- src/plotman/manager.py | 9 +++++---- src/plotman/resources/plotman.yaml | 8 ++++---- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py index 98710087..d9fbae85 100755 --- a/src/plotman/_tests/manager_test.py +++ b/src/plotman/_tests/manager_test.py @@ -13,9 +13,9 @@ def sched_cfg() -> configuration.Scheduling: global_max_jobs=1, global_stagger_m=2, polling_time_s=2, - global_tmpdir_stagger_phase_major=3, - global_tmpdir_stagger_phase_minor=0, - global_tmpdir_max_jobs=3, + tmpdir_stagger_phase_major=3, + tmpdir_stagger_phase_minor=0, + tmpdir_max_jobs=3, tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)} ) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 09e8ecc0..96c3365b 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -270,10 +270,10 @@ class Scheduling: global_max_jobs: int global_stagger_m: int polling_time_s: int - global_tmpdir_max_jobs: int - global_tmpdir_stagger_phase_major: int - global_tmpdir_stagger_phase_minor: int - global_tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 + tmpdir_max_jobs: int + tmpdir_stagger_phase_major: int + tmpdir_stagger_phase_minor: int + tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 tmp_overrides: Optional[Dict[str, TmpOverrides]] = None @attr.frozen diff --git a/src/plotman/manager.py b/src/plotman/manager.py index af1456c4..c109d577 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -54,14 +54,14 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo return True # Assign variables - major = sched_cfg.global_tmpdir_stagger_phase_major - minor = sched_cfg.global_tmpdir_stagger_phase_minor + major = sched_cfg.tmpdir_stagger_phase_major + minor = sched_cfg.tmpdir_stagger_phase_minor # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py - stagger_phase_limit = sched_cfg.global_tmpdir_stagger_phase_limit + stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit # Limit the total number of jobs per tmp dir. Default to overall max # jobs configuration, but restrict to any configured overrides. - max_plots = sched_cfg.global_tmpdir_max_jobs + max_plots = sched_cfg.tmpdir_max_jobs # Check if any overrides exist for the current job if sched_cfg.tmp_overrides is not None and d in sched_cfg.tmp_overrides: @@ -70,6 +70,7 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo # Check for and assign major & minor phase overrides if curr_overrides.tmpdir_stagger_phase_major is not None: major = curr_overrides.tmpdir_stagger_phase_major + if curr_overrides.tmpdir_stagger_phase_minor is not None: minor = curr_overrides.tmpdir_stagger_phase_minor # Check for and assign stagger phase limit override if curr_overrides.tmpdir_stagger_phase_limit is not None: diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 6212ecb2..a88dc549 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -101,13 +101,13 @@ scheduling: # e.g, with default settings, a new plot will start only when your plot # reaches phase [2 : 1] on your temp drive. This setting takes precidence # over global_stagger_m - global_tmpdir_stagger_phase_major: 2 - global_tmpdir_stagger_phase_minor: 1 + tmpdir_stagger_phase_major: 2 + tmpdir_stagger_phase_minor: 1 # Optional: default is 1 - global_tmpdir_stagger_phase_limit: 1 + tmpdir_stagger_phase_limit: 1 # Don't run more than this many jobs at a time on a single temp dir. - global_tmpdir_max_jobs: 3 + tmpdir_max_jobs: 3 # Don't run more than this many jobs at a time in total. global_max_jobs: 12 From 256f5c98fc0ecfdd99774214c5b97ff6cc2f3ffc Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 17 Jun 2021 19:56:35 -0600 Subject: [PATCH 041/109] Initial support for Madmax plotter. --- src/plotman/analyzer.py | 19 +++++++- src/plotman/configuration.py | 11 +++-- src/plotman/job.py | 73 ++++++++++++++++++++++-------- src/plotman/manager.py | 25 ++++++---- src/plotman/resources/plotman.yaml | 20 ++++---- 5 files changed, 105 insertions(+), 43 deletions(-) diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index 67073805..dcbcdc67 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -57,12 +57,18 @@ def analyze(logfilenames, clipterminals, bytmp, bybitfield): else: sl += '-bitfield' - # Phase timing. Sample log line: + # CHIA: Phase timing. Sample log line: # Time for phase 1 = 22796.7 seconds. CPU (98%) Tue Sep 29 17:57:19 2020 for phase in ['1', '2', '3', '4']: m = re.search(r'^Time for phase ' + phase + ' = (\d+.\d+) seconds..*', line) if m: phase_time[phase] = float(m.group(1)) + + # MADMAX: Phase timing. Sample log line: "Phase 2 took 2193.37 sec" + for phase in ['1', '2', '3', '4']: + m = re.search(r'^Phase ' + phase + ' took (\d+.\d+) sec.*', line) + if m: + phase_time[phase] = float(m.group(1)) # Uniform sort. Sample log line: # Bucket 267 uniform sort. Ram: 0.920GiB, u_sort min: 0.688GiB, qs min: 0.172GiB. @@ -81,7 +87,7 @@ def analyze(logfilenames, clipterminals, bytmp, bybitfield): else: print ('Warning: unrecognized sort ' + sorter) - # Job completion. Record total time in sliced data store. + # CHIA: Job completion. Record total time in sliced data store. # Sample log line: # Total time = 49487.1 seconds. CPU (97.26%) Wed Sep 30 01:22:10 2020 m = re.search(r'^Total time = (\d+.\d+) seconds.*', line) @@ -93,6 +99,15 @@ def analyze(logfilenames, clipterminals, bytmp, bybitfield): for phase in ['1', '2', '3', '4']: data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase]) data.setdefault(sl, {}).setdefault('%usort', []).append(100 * n_uniform // n_sorts) + + # MADMAX: Job completion. Record total time in sliced data store. + # Sample log line: "Total plot creation time was 2530.76 sec" + m = re.search(r'^Total plot creation time was (\d+.\d+) sec.*', line) + if m: + data.setdefault(sl, {}).setdefault('total time', []).append(float(m.group(1))) + for phase in ['1', '2', '3', '4']: + data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase]) + data.setdefault(sl, {}).setdefault('%usort', []).append('') # Not available for MADMAX # Prepare report tab = tt.Texttable() diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 220dc538..6a25c584 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -253,11 +253,12 @@ class Scheduling: @attr.frozen class Plotting: - k: int - e: bool - n_threads: int - n_buckets: int - job_buffer: int + type: str = "chia" + n_threads: int = 2 + n_buckets: int = 128 + k: Optional[int] = 32 + e: Optional[bool] = False + job_buffer: Optional[int] = 3389 farmer_pk: Optional[str] = None pool_pk: Optional[str] = None pool_contract_address: Optional[str] = None diff --git a/src/plotman/job.py b/src/plotman/job.py index e99ac07f..880e094d 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -29,14 +29,17 @@ def job_phases_for_dstdir(d, all_jobs): return sorted([j.progress() for j in all_jobs if j.dstdir == d]) def is_plotting_cmdline(cmdline): - if cmdline and 'python' in cmdline[0].lower(): + if cmdline and 'python' in cmdline[0].lower(): # Stock Chia plotter cmdline = cmdline[1:] - return ( - len(cmdline) >= 3 - and 'chia' in cmdline[0] - and 'plots' == cmdline[1] - and 'create' == cmdline[2] - ) + return ( + len(cmdline) >= 3 + and 'chia' in cmdline[0] + and 'plots' == cmdline[1] + and 'create' == cmdline[2] + ) + elif cmdline and 'chia_plot' == cmdline[0].lower(): # Madmax plotter + return True + return False def parse_chia_plot_time(s): # This will grow to try ISO8601 as well for when Chia logs that way @@ -45,14 +48,16 @@ def parse_chia_plot_time(s): def parse_chia_plots_create_command_line(command_line): command_line = list(command_line) # Parse command line args - if 'python' in command_line[0].lower(): + if 'python' in command_line[0].lower(): # Stock Chia plotter command_line = command_line[1:] - assert len(command_line) >= 3 - assert 'chia' in command_line[0] - assert 'plots' == command_line[1] - assert 'create' == command_line[2] - - all_command_arguments = command_line[3:] + assert len(command_line) >= 3 + assert 'chia' in command_line[0] + assert 'plots' == command_line[1] + assert 'create' == command_line[2] + all_command_arguments = command_line[3:] + elif 'chia_plot' in command_line[0].lower(): # Madmax plotter + command_line = command_line[1:] + all_command_arguments = command_line[2:] # nice idea, but this doesn't include -h # help_option_names = command.get_help_option_names(ctx=context) @@ -275,15 +280,22 @@ def init_from_logfile(self): with contextlib.suppress(UnicodeDecodeError): for line in f: m = re.match('^ID: ([0-9a-f]*)', line) - if m: + if m: # CHIA self.plot_id = m.group(1) found_id = True + else: + m = re.match("^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line) + if m: # MADMAX + self.plot_id = m.group(7) + found_id = True m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line) - if m: + if m: # CHIA # Mon Nov 2 08:39:53 2020 self.start_time = parse_chia_plot_time(m.group(1)) found_log = True break # Stop reading lines in file + else: # MADMAX + self.start_time = datetime.fromtimestamp(os.path.getctime(self.logfile)) if found_id and found_log: break # Stop trying @@ -316,26 +328,47 @@ def set_phase_from_logfile(self): with open(self.logfile, 'r') as f: with contextlib.suppress(UnicodeDecodeError): for line in f: - # "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" + # CHIA: "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" m = re.match(r'^Starting phase (\d).*', line) if m: phase = int(m.group(1)) phase_subphases[phase] = 0 + + # MADMAX: "[P1]" or "[P2]" or "[P3]" or "[P4]" + m = re.match(r'^\[P(\d)\].*', line) + if m: + phase = int(m.group(1)) + phase_subphases[phase] = 0 - # Phase 1: "Computing table 2" + # CHIA: Phase 1: "Computing table 2" m = re.match(r'^Computing table (\d).*', line) if m: phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) + + # MADMAX: Phase 1: "[P1] Table 2" + m = re.match(r'^\[P1\] Table (\d).*', line) + if m: + phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) - # Phase 2: "Backpropagating on table 2" + # CHIA: Phase 2: "Backpropagating on table 2" m = re.match(r'^Backpropagating on table (\d).*', line) if m: phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) - # Phase 3: "Compressing tables 4 and 5" + # MADMAX: Phase 2: "[P2] Table 2" + m = re.match(r'^\[P2\] Table (\d).*', line) + if m: + phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) + + # CHIA: Phase 3: "Compressing tables 4 and 5" m = re.match(r'^Compressing tables (\d) and (\d).*', line) if m: phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + + # MADMAX: Phase 3: "[P3-1] Table 4" + m = re.match(r'^\[P3\-\d\] Table (\d).*', line) + if m: + phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) # TODO also collect timing info: diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 7c5b116d..6575636b 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -115,29 +115,38 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg, log_cfg): log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) - plot_args = ['chia', 'plots', 'create', + if plotting_cfg.type == "madmax": + plot_args = ['chia_plot', + '-n', str(1), + '-r', str(plotting_cfg.n_threads), + '-u', str(plotting_cfg.n_buckets), + '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), + '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] + else: + plot_args = ['chia', 'plots', 'create', '-k', str(plotting_cfg.k), '-r', str(plotting_cfg.n_threads), '-u', str(plotting_cfg.n_buckets), '-b', str(plotting_cfg.job_buffer), '-t', tmpdir, '-d', dstdir ] - if plotting_cfg.e: - plot_args.append('-e') + if plotting_cfg.e: + plot_args.append('-e') + if plotting_cfg.pool_contract_address is not None: + plot_args.append('-c') + plot_args.append(plotting_cfg.pool_contract_address) + if plotting_cfg.x: + plot_args.append('-x') if plotting_cfg.farmer_pk is not None: plot_args.append('-f') plot_args.append(plotting_cfg.farmer_pk) if plotting_cfg.pool_pk is not None: plot_args.append('-p') plot_args.append(plotting_cfg.pool_pk) - if plotting_cfg.pool_contract_address is not None: - plot_args.append('-c') - plot_args.append(plotting_cfg.pool_contract_address) if dir_cfg.tmp2 is not None: plot_args.append('-2') plot_args.append(dir_cfg.tmp2) - if plotting_cfg.x: - plot_args.append('-x') + logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), log_file_path)) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 09ee52ad..d4967aef 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -136,15 +136,19 @@ scheduling: # See documentation at # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create plotting: - k: 32 + # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt + # farmer_pk: ... + # pool_pk: ... + + # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs + type: chia # The stock plotter: https://github.com/Chia-Network/chia-blockchain + k: 32 # k-size of plot, leave at 32 most of the time e: False # Use -e plotting option n_threads: 2 # Threads per job n_buckets: 128 # Number of buckets to split data into job_buffer: 3389 # Per job memory - # If specified, pass through to the -f and -p options. See CLI reference. - # farmer_pk: ... - # pool_pk: ... - # If true, Skips adding [final dir] / dst to harvester for farming. - # Especially useful if you have harvesters that are running somewhere else - # and you are just plotting on the machine where plotman is running. - # x: True + + # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs + #type: madmax # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter + #n_threads: 4 # Default is 4, crank up if you have many cores + #n_buckets: 256 # Default is 256 From 8c1ca0e9032edfa21d92c67a9ca59e2052cf73c9 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 17 Jun 2021 20:48:15 -0600 Subject: [PATCH 042/109] Guard against index error. --- src/plotman/job.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 880e094d..a257fccf 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -368,7 +368,10 @@ def set_phase_from_logfile(self): # MADMAX: Phase 3: "[P3-1] Table 4" m = re.match(r'^\[P3\-\d\] Table (\d).*', line) if m: - phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + if 3 in phase_subphases: + phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + else: + phase_subphases[3] = int(m.group(1)) # TODO also collect timing info: From fb4a05cd866a3348d171f19ea767c21ead285602 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Fri, 18 Jun 2021 07:05:14 -0600 Subject: [PATCH 043/109] Madmax doesn't seem to output sorting info. --- src/plotman/analyzer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index 735e1c50..b7229fe2 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -108,14 +108,14 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by data.setdefault(sl, {}).setdefault('total time', []).append(float(m.group(1))) for phase in ['1', '2', '3', '4']: data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase]) - data.setdefault(sl, {}).setdefault('%usort', []).append('') # Not available for MADMAX + data.setdefault(sl, {}).setdefault('%usort', []).append(0) # Not available for MADMAX # Prepare report tab = tt.Texttable() all_measures = ['%usort', 'phase 1', 'phase 2', 'phase 3', 'phase 4', 'total time'] headings = ['Slice', 'n'] + all_measures tab.header(headings) - + for sl in data.keys(): row = [sl] From 8163dac106d0930dfc6b835b96db6daf724d395f Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Fri, 18 Jun 2021 17:54:55 +0100 Subject: [PATCH 044/109] Remove unnecessary comma Co-authored-by: Kyle Altendorf --- src/plotman/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index c109d577..a63bf7a2 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -79,7 +79,7 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo if curr_overrides.tmpdir_max_jobs is not None: max_plots = curr_overrides.tmpdir_max_jobs - milestone = job.Phase(major,minor,) + milestone = job.Phase(major,minor) # Check if phases pass the criteria if len([p for p in phases if p < milestone]) >= stagger_phase_limit or len(phases) >= max_plots: From 64a8eb75a82e4534973366aac641e24c195a5993 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Fri, 18 Jun 2021 17:55:16 +0100 Subject: [PATCH 045/109] Update CHANGELOG.md Co-authored-by: Kyle Altendorf --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 744c5845..f0127041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Added - optional tmpdir_overrides added for each specified tmpdir -- ([#758](https://github.com/ericaltendorf/plotman/pull/758)) + ([#758](https://github.com/ericaltendorf/plotman/pull/758)) - `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) - `--json` option for `plotman status`. From f06400e5fe9122e56c41d112c841b29d60fcc8ab Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Fri, 18 Jun 2021 17:57:06 +0100 Subject: [PATCH 046/109] Undo change Co-authored-by: Kyle Altendorf --- src/plotman/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index a63bf7a2..fa544b12 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -97,7 +97,7 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: if (youngest_job_age < global_stagger): wait_reason = 'global stagger (%ds/%ds)' % (youngest_job_age, global_stagger) elif len(jobs) >= sched_cfg.global_max_jobs: - wait_reason = 'global max jobs reached (%d)' % (sched_cfg.global_max_jobs) + wait_reason = 'global max jobs reached (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger) else: tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp] eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases From 69fed66332db1e5b0bc184f85678bc697c9f73c0 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Fri, 18 Jun 2021 18:01:18 +0100 Subject: [PATCH 047/109] Update src/plotman/manager.py Co-authored-by: Kyle Altendorf --- src/plotman/manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index fa544b12..24f2b965 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -82,7 +82,10 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo milestone = job.Phase(major,minor) # Check if phases pass the criteria - if len([p for p in phases if p < milestone]) >= stagger_phase_limit or len(phases) >= max_plots: + if len([p for p in phases if p < milestone]) >= stagger_phase_limit + return False + + if len(phases) >= max_plots: return False else: return True From ad0b9e07bd0cd35350e126c88c3dea01981fd018 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Fri, 18 Jun 2021 18:01:25 +0100 Subject: [PATCH 048/109] Update src/plotman/manager.py Co-authored-by: Kyle Altendorf --- src/plotman/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 24f2b965..b7f3a2a6 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -87,8 +87,8 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo if len(phases) >= max_plots: return False - else: - return True + + return True def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: plotman.configuration.Scheduling, plotting_cfg: plotman.configuration.Plotting, log_cfg: plotman.configuration.Logging) -> typing.Tuple[bool, str]: jobs = job.Job.get_running_jobs(log_cfg.plots) From e25b28753646e064f5f5821d5a09be0ba1154cd7 Mon Sep 17 00:00:00 2001 From: Martin Ruckli Date: Fri, 18 Jun 2021 23:47:30 +0200 Subject: [PATCH 049/109] adds logs printing command --- src/plotman/job.py | 18 +++++++++++++++++- src/plotman/plotman.py | 10 +++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index b5c4a838..d065f326 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -412,7 +412,23 @@ def status_str_long(self) -> str: dst = self.dstdir, logfile = self.logfile ) - + + def print_logs(self, follow=None): + with open(self.logfile, 'r') as f: + if follow: + line = '' + while True: + tmp = f.readline() + if tmp is not None: + line += tmp + if line.endswith("\n"): + print(line.rstrip('\n')) + line = '' + else: + time.sleep(0.1) + else: + print(f.read()) + def to_dict(self) -> typing.Dict[str, object]: '''Exports important information as dictionary.''' return dict( diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index a844905a..06cfeded 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -62,6 +62,11 @@ def parse_args(self) -> typing.Any: p_details = sp.add_parser('details', help='show details for job') self.add_idprefix_arg(p_details) + p_logs = sp.add_parser('logs', help='fetch the logs for job') + + p_logs.add_argument('-f', '--follow', action='store_true', help='Follow log output') + self.add_idprefix_arg(p_logs) + p_files = sp.add_parser('files', help='show temp files associated with job') self.add_idprefix_arg(p_files) @@ -261,7 +266,7 @@ def main() -> None: # # Job control commands # - elif args.cmd in [ 'details', 'files', 'kill', 'suspend', 'resume' ]: + elif args.cmd in [ 'details', 'logs', 'files', 'kill', 'suspend', 'resume' ]: print(args) selected = [] @@ -284,6 +289,9 @@ def main() -> None: if args.cmd == 'details': print(job.status_str_long()) + elif args.cmd == 'logs': + job.print_logs(args.follow) + elif args.cmd == 'files': temp_files = job.get_temp_files() for f in temp_files: From d2c9f081adb1b57668e13e41478051f9b825ade2 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 18 Jun 2021 22:11:22 -0400 Subject: [PATCH 050/109] add type hints --- src/plotman/_tests/reporting_test.py | 2 +- src/plotman/reporting.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plotman/_tests/reporting_test.py b/src/plotman/_tests/reporting_test.py index e566f3e6..5ce0ffa9 100644 --- a/src/plotman/_tests/reporting_test.py +++ b/src/plotman/_tests/reporting_test.py @@ -62,7 +62,7 @@ def test_job_viz_counts() -> None: assert(reporting.job_viz(jobs) == '1 2 .:;! 3 ! 4 ') # type: ignore[arg-type] -def test_to_prometheus_format(): +def test_to_prometheus_format() -> None: prom_stati = [ ('foo="bar",baz="2"', {'metric1': 1, 'metric2': 2}), ('foo="blubb",baz="3"', {'metric1': 2, 'metric2': 3}) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 46ef98eb..0a74d84d 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -124,7 +124,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona return tab.draw() # type: ignore[no-any-return] -def to_prometheus_format(metrics, prom_stati): +def to_prometheus_format(metrics: typing.Dict[str, str], prom_stati: typing.Sequence[typing.Tuple[str, typing.Mapping[str, typing.Optional[int]]]]) -> typing.List[str]: prom_str_list = [] for metric_name, metric_desc in metrics.items(): prom_str_list.append(f'# HELP {metric_name} {metric_desc}.') @@ -133,7 +133,7 @@ def to_prometheus_format(metrics, prom_stati): prom_str_list.append('%s{%s} %s' % (metric_name, label_str, values[metric_name])) return prom_str_list -def prometheus_report(jobs, tmp_prefix='', dst_prefix=''): +def prometheus_report(jobs: typing.List[job.Job], tmp_prefix: str = '', dst_prefix: str = '') -> str: metrics = { 'plotman_plot_phase_major': 'The phase the plot is currently in', 'plotman_plot_phase_minor': 'The part of the phase the plot is currently in', @@ -150,7 +150,7 @@ def prometheus_report(jobs, tmp_prefix='', dst_prefix=''): 'tmp_dir': abbr_path(j.tmpdir, tmp_prefix), 'dst_dir': abbr_path(j.dstdir, dst_prefix), 'run_status': j.get_run_status(), - 'phase': phase_str(j.progress()) + 'phase': str(j.progress()), } label_str = ','.join([f'{k}="{v}"' for k, v in labels.items()]) values = { From d92f397f0d673b3759aaaf04c6b07261b97ec03f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 18 Jun 2021 22:25:38 -0400 Subject: [PATCH 051/109] add changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b291813..b03cf8ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#549](https://github.com/ericaltendorf/plotman/pull/549)) - If the tmp drive selected for a plot is also listed as a dst drive then plotman will use the same drive for both. ([#643](https://github.com/ericaltendorf/plotman/pull/643)) +- `plotman prometheus` command to output status for consumption by [Prometheus](https://prometheus.io/). + ([#430](https://github.com/ericaltendorf/plotman/pull/430)) ## [0.4.1] - 2021-06-11 ### Fixed From 3a92068aa54decf19b9afe4046c4f507f52f13b4 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Sat, 19 Jun 2021 16:02:49 +0100 Subject: [PATCH 052/109] Add missing colon --- src/plotman/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index b7f3a2a6..c204d00f 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -82,7 +82,7 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo milestone = job.Phase(major,minor) # Check if phases pass the criteria - if len([p for p in phases if p < milestone]) >= stagger_phase_limit + if len([p for p in phases if p < milestone]) >= stagger_phase_limit: return False if len(phases) >= max_plots: From b74e2ad938dcd996b78d4cc6ca5e5dead450433c Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Sun, 20 Jun 2021 18:26:21 +0100 Subject: [PATCH 053/109] Return wait_reason to orig --- src/plotman/manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index c204d00f..f7d5e8b1 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -98,9 +98,9 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: youngest_job_age = min(jobs, key=job.Job.get_time_wall).get_time_wall() if jobs else MAX_AGE global_stagger = int(sched_cfg.global_stagger_m * MIN) if (youngest_job_age < global_stagger): - wait_reason = 'global stagger (%ds/%ds)' % (youngest_job_age, global_stagger) + wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger) elif len(jobs) >= sched_cfg.global_max_jobs: - wait_reason = 'global max jobs reached (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger) + wait_reason = 'max jobs (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger) else: tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp] eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases @@ -109,7 +109,7 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: for (d, phases) in eligible ] if not eligible: - wait_reason = 'waiting for phase match' + wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)' else: # Plot to oldest tmpdir. tmpdir = max(rankable, key=operator.itemgetter(1))[0] From 3c6d8fa25c16b4487f9190cbf70b5965e949bb96 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Sun, 20 Jun 2021 18:32:52 +0100 Subject: [PATCH 054/109] abolish aberrant apostrophe! --- src/plotman/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index f7d5e8b1..88d18127 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -109,7 +109,7 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: for (d, phases) in eligible ] if not eligible: - wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)' + wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger) else: # Plot to oldest tmpdir. tmpdir = max(rankable, key=operator.itemgetter(1))[0] From 41a4ab28c3f3b721c809c48ca29cab46f841050c Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Sun, 20 Jun 2021 13:43:34 -0600 Subject: [PATCH 055/109] Improvements for code review. --- CHANGELOG.md | 2 ++ src/plotman/job.py | 3 +++ src/plotman/reporting.py | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b03cf8ad..badb9e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#643](https://github.com/ericaltendorf/plotman/pull/643)) - `plotman prometheus` command to output status for consumption by [Prometheus](https://prometheus.io/). ([#430](https://github.com/ericaltendorf/plotman/pull/430)) +- Initial support for MadMax plotter. + ([#797](https://github.com/ericaltendorf/plotman/pull/797)) ## [0.4.1] - 2021-06-11 ### Fixed diff --git a/src/plotman/job.py b/src/plotman/job.py index 34db4d25..0fe3fdc1 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -151,6 +151,7 @@ class Job: jobfile: str = '' job_id: int = 0 plot_id: str = '--------' + plotter: str = '' proc: psutil.Process k: int r: int @@ -318,11 +319,13 @@ def init_from_logfile(self) -> None: m = re.match('^ID: ([0-9a-f]*)', line) if m: # CHIA self.plot_id = m.group(1) + self.plotter = 'chia' found_id = True else: m = re.match("^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line) if m: # MADMAX self.plot_id = m.group(7) + self.plotter = 'madmax' found_id = True m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line) if m: # CHIA diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 0a74d84d..ac759810 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -75,7 +75,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona n_end_rows = n_rows - n_begin_rows tab = tt.Texttable() - headings = ['plot id', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp', + headings = ['plot id', 'plotter', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp', 'pid', 'stat', 'mem', 'user', 'sys', 'io'] if height: headings.insert(0, '#') @@ -97,6 +97,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona try: with j.proc.oneshot(): row = [j.plot_id[:8], # Plot ID + str(j.plotter), # chia or madmax str(j.k), # k size abbr_path(j.tmpdir, tmp_prefix), # Temp directory abbr_path(j.dstdir, dst_prefix), # Destination directory From b1b4899b64246dfe44fe701de02581efc7c43a85 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 20 Jun 2021 18:16:53 -0700 Subject: [PATCH 056/109] Update src/plotman/job.py --- src/plotman/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index d065f326..47c649f4 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -413,7 +413,7 @@ def status_str_long(self) -> str: logfile = self.logfile ) - def print_logs(self, follow=None): + def print_logs(self, follow: bool = False) -> None: with open(self.logfile, 'r') as f: if follow: line = '' From ae5f45e478fc4e5c75063cdee6a9f0fd82bd729d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 20 Jun 2021 21:19:58 -0400 Subject: [PATCH 057/109] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b03cf8ad..ab0c93c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#643](https://github.com/ericaltendorf/plotman/pull/643)) - `plotman prometheus` command to output status for consumption by [Prometheus](https://prometheus.io/). ([#430](https://github.com/ericaltendorf/plotman/pull/430)) +- `plotman logs` command to print and tail plot logs by their plot ID. + ([#509](https://github.com/ericaltendorf/plotman/pull/509)) ## [0.4.1] - 2021-06-11 ### Fixed From 169def83b4af0a8107f484dd940c1261f44ed928 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Mon, 21 Jun 2021 10:45:12 -0600 Subject: [PATCH 058/109] Update src/plotman/job.py Co-authored-by: Kyle Altendorf --- src/plotman/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index d27cb6ee..d659c92e 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -334,7 +334,7 @@ def init_from_logfile(self) -> None: found_log = True break # Stop reading lines in file else: # MADMAX - self.start_time = datetime.fromtimestamp(os.path.getctime(self.logfile)) + self.start_time = pendulum.from_timestamp(os.path.getctime(self.logfile)) if found_id and found_log: break # Stop trying From 2761c6851a08b0f13e0d74c691c7aca04e699a97 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Mon, 21 Jun 2021 10:48:28 -0600 Subject: [PATCH 059/109] Update src/plotman/manager.py Co-authored-by: Kyle Altendorf --- src/plotman/manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index a3855ba8..80d31db7 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -121,15 +121,16 @@ def key(key: str) -> job.Phase: log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) + plot_args: typing.List[str] if plotting_cfg.type == "madmax": - plot_args: typing.List[str] = ['chia_plot', + plot_args = ['chia_plot', '-n', str(1), '-r', str(plotting_cfg.n_threads), '-u', str(plotting_cfg.n_buckets), '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] else: - plot_args: typing.List[str] = ['chia', 'plots', 'create', + plot_args = ['chia', 'plots', 'create', '-k', str(plotting_cfg.k), '-r', str(plotting_cfg.n_threads), '-u', str(plotting_cfg.n_buckets), From 8b7e7ea48a366c2bc83f0a08b828a73d43a91749 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Mon, 21 Jun 2021 11:02:21 -0600 Subject: [PATCH 060/109] Madmax CLI options mapped to Click. --- src/plotman/madmax.py | 72 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/plotman/madmax.py diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py new file mode 100644 index 00000000..79476ad7 --- /dev/null +++ b/src/plotman/madmax.py @@ -0,0 +1,72 @@ +# mypy: allow_untyped_decorators +# +# Madmax is written in C++. Below is a mapping of its CLI options to Python. +# See: https://github.com/madMAx43v3r/chia-plotter/tree/master/src +# Note: versions are git commit refs, not semantic versioning +# + +import functools +import typing + +import click +from pathlib import Path +import typing_extensions + + +class CommandProtocol(typing_extensions.Protocol): + def make_context(self, info_name: str, args: typing.List[str]) -> click.Context: + ... + + def __call__(self) -> None: + ... + + +class Commands: + def __init__(self) -> None: + self.by_version: typing.Dict[typing.Sequence[int], CommandProtocol] = {} + + def register(self, version: typing.Sequence[int]) -> typing.Callable[[CommandProtocol], None]: + if version in self.by_version: + raise Exception(f'Version already registered: {version!r}') + + return functools.partial(self._decorator, version=version) + + def _decorator(self, command: CommandProtocol, *, version: typing.Sequence[int]) -> None: + self.by_version[version] = command + + def __getitem__(self, item: typing.Sequence[int]) -> typing.Callable[[], None]: + return self.by_version[item] + + def latest_command(self) -> CommandProtocol: + return max(self.by_version.items())[1] + + +commands = Commands() +# Madmax Git on 2021-06-19 -> https://github.com/madMAx43v3r/chia-plotter/commit/c8121b987186c42c895b49818e6c13acecc51332 +@commands.register(version=("c8121b9")) +@click.command() +# https://github.com/madMAx43v3r/chia-plotter/blob/master/LICENSE +# https://github.com/madMAx43v3r/chia-plotter/blob/master/src/chia_plot.cpp#L180 +@click.option("-n", "--count", help="Number of plots to create (default = 1, -1 = infinite)", + type=int, default=1, show_default=True) +@click.option("-r", "--threads", help="Number of threads (default = 4)", + type=int, default=4, show_default=True) +@click.option("-u", "--buckets", help="Number of buckets (default = 256)", + type=int, default=256, show_default=True) +@click.option("-v", "--buckets3", help="Number of buckets for phase 3+4 (default = buckets)", + type=int, default=256) +@click.option("-t", "--tmpdir", help="Temporary directory, needs ~220 GiB (default = $PWD)", + type=click.Path(), default=Path("."), show_default=True) +@click.option("-2", "--tmpdir2", help="Temporary directory 2, needs ~110 GiB [RAM] (default = )", + type=click.Path(), default=None) +@click.option("-d", "--finaldir", help="Final directory (default = )", + type=click.Path(), default=Path("."), show_default=True) +@click.option("-p", "--poolkey", help="Pool Public Key (48 bytes)", + type=str, default=None) +@click.option("-f", "--farmerkey", help="Farmer Public Key (48 bytes)", + type=str, default=None) +@click.option("-G", "--tmptoggle", help="Alternate tmpdir/tmpdir2", + type=str, default=None) +) +def _cli_c8121b9() -> None: + pass From 4ef83e8d9c368882efc78919f9a124361f8ea3b6 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 21:24:12 -0400 Subject: [PATCH 061/109] set the required configuration major version to 2 there are multiple incoming PRs with configuration changes --- src/plotman/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 3181b5a9..fd768b27 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -49,7 +49,7 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi version = config_objects.get('version', (0,)) - expected_major_version = 1 + expected_major_version = 2 if version[0] != expected_major_version: message = textwrap.dedent(f"""\ From 9020a3983f1aad986d28552798536a1b8eac76a7 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 21:26:17 -0400 Subject: [PATCH 062/109] update example configuration version --- src/plotman/resources/plotman.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 09ee52ad..899ac07e 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -1,7 +1,7 @@ # Default/example plotman.yaml configuration file # https://github.com/ericaltendorf/plotman/wiki/Configuration#versions -version: [1] +version: [2] logging: # One directory in which to store all plot job logs (the STDOUT/ From 4c1978b8d9efffdac772ff8c3dca0cb640100368 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 21:54:32 -0400 Subject: [PATCH 063/109] remove extra ) --- src/plotman/madmax.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py index 79476ad7..21a429c0 100644 --- a/src/plotman/madmax.py +++ b/src/plotman/madmax.py @@ -67,6 +67,5 @@ def latest_command(self) -> CommandProtocol: type=str, default=None) @click.option("-G", "--tmptoggle", help="Alternate tmpdir/tmpdir2", type=str, default=None) -) def _cli_c8121b9() -> None: pass From 7b1572f19c8626057d63b6d2965519bccc883212 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 22:00:51 -0400 Subject: [PATCH 064/109] use madmax cli parsing standin --- src/plotman/job.py | 11 ++++++----- src/plotman/madmax.py | 4 +++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index d659c92e..01f40ffe 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -18,7 +18,7 @@ import pendulum import psutil -from plotman import chia +from plotman import chia, madmax def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: @@ -60,9 +60,14 @@ def parse_chia_plots_create_command_line( assert 'plots' == command_line[1] assert 'create' == command_line[2] all_command_arguments = command_line[3:] + # TODO: We could at some point do chia version detection and pick the + # associated command. For now we'll just use the latest one we have + # copied. + command = chia.commands.latest_command() elif 'chia_plot' in command_line[0].lower(): # Madmax plotter command_line = command_line[1:] all_command_arguments = command_line[2:] + command = madmax._cli_c8121b9 # nice idea, but this doesn't include -h # help_option_names = command.get_help_option_names(ctx=context) @@ -74,10 +79,6 @@ def parse_chia_plots_create_command_line( if argument not in help_option_names ] - # TODO: We could at some point do chia version detection and pick the - # associated command. For now we'll just use the latest one we have - # copied. - command = chia.commands.latest_command() try: context = command.make_context(info_name='', args=list(command_arguments)) except click.ClickException as e: diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py index 21a429c0..c6ba0626 100644 --- a/src/plotman/madmax.py +++ b/src/plotman/madmax.py @@ -43,7 +43,9 @@ def latest_command(self) -> CommandProtocol: commands = Commands() # Madmax Git on 2021-06-19 -> https://github.com/madMAx43v3r/chia-plotter/commit/c8121b987186c42c895b49818e6c13acecc51332 -@commands.register(version=("c8121b9")) +# TODO: make Commands able to handle this. maybe configure with a list defining order? +# for now we can just access directly. +# @commands.register(version=("c8121b9")) @click.command() # https://github.com/madMAx43v3r/chia-plotter/blob/master/LICENSE # https://github.com/madMAx43v3r/chia-plotter/blob/master/src/chia_plot.cpp#L180 From 31efe1818408d8f2f39f4545c7ad3820e01a2928 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 23:05:05 -0400 Subject: [PATCH 065/109] separate chia and madmax plotter configuration sections --- src/plotman/configuration.py | 51 +++++++++++++++++++++++++++--- src/plotman/manager.py | 28 ++++++++++------ src/plotman/resources/plotman.yaml | 35 ++++++++++---------- 3 files changed, 84 insertions(+), 30 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 1c393057..29334547 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -12,6 +12,8 @@ # TODO: should be a desert.ib() but mypy doesn't understand it then, see below import desert._make import marshmallow +import marshmallow.fields +import marshmallow.validate import pendulum import yaml @@ -67,6 +69,27 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi f"Config file at: '{config_path}' is malformed" ) from e + if loaded.plotting.type == "chia" and loaded.plotting.chia is None: + raise ConfigurationException( + "chia selected as plotter but plotting: chia: was not specified in the config", + ) + elif loaded.plotting.type == "madmax": + if loaded.plotting.madmax is None: + raise ConfigurationException( + "madmax selected as plotter but plotting: madmax: was not specified in the config", + ) + + if loaded.plotting.farmer_pk is None: + raise ConfigurationException( + "madmax selected as plotter but no plotting: farmer_pk: was specified", + ) + + if loaded.plotting.pool_pk is None: + raise ConfigurationException( + "madmax selected as plotter but no plotting: pool_pk: was specified", + ) + + if loaded.archiving is not None: preset_target_objects = yaml.safe_load(preset_target_definitions_text) preset_target_schema = desert.schema(PresetTargetDefinitions) @@ -273,18 +296,38 @@ class Scheduling: tmpdir_stagger_phase_minor: int tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 + @attr.frozen -class Plotting: - type: str = "chia" +class ChiaPlotterOptions: n_threads: int = 2 n_buckets: int = 128 k: Optional[int] = 32 e: Optional[bool] = False job_buffer: Optional[int] = 3389 + x: bool = False + pool_contract_address: Optional[str] = None + +@attr.frozen +class MadmaxPlotterOptions: + n_threads: int = 4 + n_buckets: int = 256 + +@attr.frozen +class Plotting: farmer_pk: Optional[str] = None pool_pk: Optional[str] = None - pool_contract_address: Optional[str] = None - x: bool = False + type: str = attr.ib( + default="chia", + metadata={ + desert._make._DESERT_SENTINEL: { + 'marshmallow_field': marshmallow.fields.String( + validate=marshmallow.validate.OneOf(choices=["chia", "madmax"]), + ), + }, + }, + ) + chia: Optional[ChiaPlotterOptions] = None + madmax: Optional[MadmaxPlotterOptions] = None @attr.frozen class UserInterface: diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 80d31db7..4ca6068e 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -123,26 +123,34 @@ def key(key: str) -> job.Phase: plot_args: typing.List[str] if plotting_cfg.type == "madmax": + if plotting_cfg.madmax is None: + raise Exception( + "madmax plotter selected but not configured, report this as a plotman bug", + ) plot_args = ['chia_plot', '-n', str(1), - '-r', str(plotting_cfg.n_threads), - '-u', str(plotting_cfg.n_buckets), + '-r', str(plotting_cfg.madmax.n_threads), + '-u', str(plotting_cfg.madmax.n_buckets), '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] else: + if plotting_cfg.chia is None: + raise Exception( + "chia plotter selected but not configured, report this as a plotman bug", + ) plot_args = ['chia', 'plots', 'create', - '-k', str(plotting_cfg.k), - '-r', str(plotting_cfg.n_threads), - '-u', str(plotting_cfg.n_buckets), - '-b', str(plotting_cfg.job_buffer), + '-k', str(plotting_cfg.chia.k), + '-r', str(plotting_cfg.chia.n_threads), + '-u', str(plotting_cfg.chia.n_buckets), + '-b', str(plotting_cfg.chia.job_buffer), '-t', tmpdir, '-d', dstdir ] - if plotting_cfg.e: + if plotting_cfg.chia.e: plot_args.append('-e') - if plotting_cfg.pool_contract_address is not None: + if plotting_cfg.chia.pool_contract_address is not None: plot_args.append('-c') - plot_args.append(plotting_cfg.pool_contract_address) - if plotting_cfg.x: + plot_args.append(plotting_cfg.chia.pool_contract_address) + if plotting_cfg.chia.x: plot_args.append('-x') if plotting_cfg.farmer_pk is not None: plot_args.append('-f') diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 573254a0..b1b35981 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -132,23 +132,26 @@ scheduling: polling_time_s: 20 -# Plotting parameters. These are pass-through parameters to chia plots create. -# See documentation at -# https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create +# Plotting parameters. These are pass-through parameters to the selected plotter. plotting: - # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt - # farmer_pk: ... - # pool_pk: ... - # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs - type: chia # The stock plotter: https://github.com/Chia-Network/chia-blockchain - k: 32 # k-size of plot, leave at 32 most of the time - e: False # Use -e plotting option - n_threads: 2 # Threads per job - n_buckets: 128 # Number of buckets to split data into - job_buffer: 3389 # Per job memory + type: chia + chia: + # The stock plotter: https://github.com/Chia-Network/chia-blockchain + # See documentation at + # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create + # farmer_pk: ... + # pool_pk: ... + k: 32 # k-size of plot, leave at 32 most of the time + e: False # Use -e plotting option + n_threads: 2 # Threads per job + n_buckets: 128 # Number of buckets to split data into + job_buffer: 3389 # Per job memory # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs - #type: madmax # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter - #n_threads: 4 # Default is 4, crank up if you have many cores - #n_buckets: 256 # Default is 256 + madmax: + # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter + farmer_pk: abc123 + pool_pk: abc123 + n_threads: 4 # Default is 4, crank up if you have many cores + n_buckets: 256 # Default is 256 From 9d5c976912824fad6c800e7068631ad7e20f0a7d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 23:08:38 -0400 Subject: [PATCH 066/109] update configuration example --- src/plotman/configuration.py | 5 ++--- src/plotman/resources/plotman.yaml | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 29334547..6d59b936 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -81,12 +81,12 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi if loaded.plotting.farmer_pk is None: raise ConfigurationException( - "madmax selected as plotter but no plotting: farmer_pk: was specified", + "madmax selected as plotter but no plotting: farmer_pk: was specified in the config", ) if loaded.plotting.pool_pk is None: raise ConfigurationException( - "madmax selected as plotter but no plotting: pool_pk: was specified", + "madmax selected as plotter but no plotting: pool_pk: was specified in the config", ) @@ -296,7 +296,6 @@ class Scheduling: tmpdir_stagger_phase_minor: int tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 - @attr.frozen class ChiaPlotterOptions: n_threads: int = 2 diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index b1b35981..49042676 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -132,16 +132,18 @@ scheduling: polling_time_s: 20 -# Plotting parameters. These are pass-through parameters to the selected plotter. +# Plotting parameters. These are pass-through parameters to chia plots create. +# See documentation at +# https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create plotting: + # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt + # farmer_pk: ... + # pool_pk: ... + # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs type: chia chia: # The stock plotter: https://github.com/Chia-Network/chia-blockchain - # See documentation at - # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create - # farmer_pk: ... - # pool_pk: ... k: 32 # k-size of plot, leave at 32 most of the time e: False # Use -e plotting option n_threads: 2 # Threads per job @@ -151,7 +153,5 @@ plotting: # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs madmax: # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter - farmer_pk: abc123 - pool_pk: abc123 n_threads: 4 # Default is 4, crank up if you have many cores n_buckets: 256 # Default is 256 From 0dfcaf4d19a2ab03274018479cf1abc06e67effc Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 22 Jun 2021 06:14:29 -0600 Subject: [PATCH 067/109] Handle two sets of args. Tmp2 trailing slash fix. --- src/plotman/job.py | 27 ++++++++++++++++++--------- src/plotman/manager.py | 10 +++++++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 01f40ffe..4c60998c 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -267,15 +267,24 @@ def __init__( # 'nobitfield': False, # 'exclude_final_dir': False, # } - - self.k = self.args['size'] # type: ignore[assignment] - self.r = self.args['num_threads'] # type: ignore[assignment] - self.u = self.args['buckets'] # type: ignore[assignment] - self.b = self.args['buffer'] # type: ignore[assignment] - self.n = self.args['num'] # type: ignore[assignment] - self.tmpdir = self.args['tmp_dir'] # type: ignore[assignment] - self.tmp2dir = self.args['tmp2_dir'] # type: ignore[assignment] - self.dstdir = self.args['final_dir'] # type: ignore[assignment] + if proc.name().startswith("chia_plot"): # MADMAX + self.k = 32 # type: ignore[assignment] + self.r = self.args['threads'] # type: ignore[assignment] + self.u = self.args['buckets'] # type: ignore[assignment] + self.b = 0 # type: ignore[assignment] + self.n = self.args['count'] # type: ignore[assignment] + self.tmpdir = self.args['tmpdir'] # type: ignore[assignment] + self.tmp2dir = self.args['tmpdir2'] # type: ignore[assignment] + self.dstdir = self.args['finaldir'] # type: ignore[assignment] + else: # CHIA + self.k = self.args['size'] # type: ignore[assignment] + self.r = self.args['num_threads'] # type: ignore[assignment] + self.u = self.args['buckets'] # type: ignore[assignment] + self.b = self.args['buffer'] # type: ignore[assignment] + self.n = self.args['num'] # type: ignore[assignment] + self.tmpdir = self.args['tmp_dir'] # type: ignore[assignment] + self.tmp2dir = self.args['tmp2_dir'] # type: ignore[assignment] + self.dstdir = self.args['final_dir'] # type: ignore[assignment] plot_cwd: str = self.proc.cwd() self.tmpdir = os.path.join(plot_cwd, self.tmpdir) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 4ca6068e..40d07b93 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -133,6 +133,9 @@ def key(key: str) -> job.Phase: '-u', str(plotting_cfg.madmax.n_buckets), '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] + if dir_cfg.tmp2 is not None: + plot_args.append('-2') + plot_args.append(dir_cfg.tmp2 if dir_cfg.tmp2.endswith('/') else (dir_cfg.tmp2 + '/')) else: if plotting_cfg.chia is None: raise Exception( @@ -152,15 +155,16 @@ def key(key: str) -> job.Phase: plot_args.append(plotting_cfg.chia.pool_contract_address) if plotting_cfg.chia.x: plot_args.append('-x') + if dir_cfg.tmp2 is not None: + plot_args.append('-2') + plot_args.append(dir_cfg.tmp2) if plotting_cfg.farmer_pk is not None: plot_args.append('-f') plot_args.append(plotting_cfg.farmer_pk) if plotting_cfg.pool_pk is not None: plot_args.append('-p') plot_args.append(plotting_cfg.pool_pk) - if dir_cfg.tmp2 is not None: - plot_args.append('-2') - plot_args.append(dir_cfg.tmp2) + logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), log_file_path)) From e364d757edb7b1042bcc5c8b5d775cd97b548641 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Tue, 22 Jun 2021 20:00:28 +0100 Subject: [PATCH 068/109] Incorporate review comments --- Dockerfile | 28 ++++++++++++---------------- build-docker-plotman.sh | 15 ++++++++------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4435ec98..58908986 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,39 +1,36 @@ # Build deployable artifacts -FROM ubuntu:latest as plotman-builder +ARG BASE_CONTAINER=ubuntu:20.04 +FROM ${BASE_CONTAINER} as plotman-builder -ARG CHIA_BRANCH +ARG CHIA_GIT_REFERENCE -RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 ansible tar bash ca-certificates git openssl unzip wget python3-pip sudo acl build-essential python3-dev python3.8-venv python3.8-distutils apt nfs-common python-is-python3 cmake libsodium-dev g++ +RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 ansible tar bash ca-certificates git openssl unzip wget python3-pip sudo acl build-essential python3-dev python3.8-venv python3.8-distutils apt nfs-common python-is-python3 -RUN echo "cloning ${CHIA_BRANCH}" -RUN git clone --branch ${CHIA_BRANCH} https://github.com/Chia-Network/chia-blockchain.git \ +RUN echo "cloning ${CHIA_GIT_REFERENCE}" +RUN git clone --branch "${CHIA_GIT_REFERENCE}" https://github.com/Chia-Network/chia-blockchain.git \ && cd chia-blockchain \ -&& git submodule update --init mozilla-ca \ -&& chmod +x install.sh +&& git submodule update --init mozilla-ca WORKDIR /chia-blockchain # Placeholder for patches -RUN /usr/bin/sh ./install.sh +RUN /bin/bash ./install.sh COPY . /plotman -RUN . ./activate \ -&& pip install -e /plotman \ -&& deactivate +RUN ["/bin/bash", "-c", "source ./activate && pip install /plotman && deactivate"] # Build deployment container -FROM ubuntu:latest as plotman +FROM ${BASE_CONTAINER} as plotman ARG UID=10001 ARG GID=10001 RUN DEBIAN_FRONTEND=noninteractive apt-get update \ -&& DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 python3.8-venv python3.8-distutils ca-certificates tzdata vim ssh less rsync git tmux libsodium23 \ +&& DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 python3.8-venv ca-certificates tzdata ssh rsync \ && apt-get clean all \ && rm -rf /var/lib/apt/lists COPY --from=plotman-builder /chia-blockchain /chia-blockchain -COPY --from=plotman-builder /plotman /plotman RUN groupadd -g ${GID} chia RUN useradd -m -u ${UID} -g ${GID} chia @@ -45,7 +42,6 @@ RUN mkdir -p /data/chia/tmp \ VOLUME ["/data/chia/tmp","/data/chia/plots","/data/chia/logs"] RUN chown -R chia:chia /chia-blockchain \ -&& chown -R chia:chia /plotman \ && chown -R chia:chia /data/chia WORKDIR /chia-blockchain @@ -57,4 +53,4 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" # Kick off plots (assumes the environemnt is good to go) CMD ["/bin/bash", "-c", "plotman plot" ] # Alternative command to simply provide shell environment -#CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ] +# CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ] diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh index 7e9a3be7..61809a11 100755 --- a/build-docker-plotman.sh +++ b/build-docker-plotman.sh @@ -1,20 +1,21 @@ -#!/bin/sh +#!/bin/bash -DOCKER_REGISTRY="graemes" -LOCAL_REGISTRY="registry.graemes.com/graemes" +LOCAL_REGISTRY="" +#DOCKER_REGISTRY="" PROJECT="chia-plotman" TAG="plotter" -CHIA_BRANCH="1.1.7" +BASE_CONTAINER="ubuntu:20.04" +CHIA_GIT_REFERENCE="1.1.7" docker rmi ${LOCAL_REGISTRY}/${PROJECT}:${TAG} docker build . \ --squash \ - --build-arg CHIA_BRANCH=${CHIA_BRANCH} \ + --build-arg BASE_CONTAINER=${BASE_CONTAINER} \ + --build-arg CHIA_GIT_REFERENCE=${CHIA_GIT_REFERENCE} \ -f Dockerfile \ -t ${LOCAL_REGISTRY}/${PROJECT}:${TAG} - -# -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} \ +# -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} docker push ${LOCAL_REGISTRY}/${PROJECT}:${TAG} #docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG} From 261d91c5077f393bb9c8c1babd6304abe3b22acd Mon Sep 17 00:00:00 2001 From: Pablo Garcia de los Salmones Valencia Date: Wed, 23 Jun 2021 01:58:50 +0200 Subject: [PATCH 069/109] Find generalized plots from k32 to any k value --- src/plotman/plot_util.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py index 37215527..5238f5e4 100644 --- a/src/plotman/plot_util.py +++ b/src/plotman/plot_util.py @@ -55,17 +55,18 @@ def split_path_prefix(items: typing.List[str]) -> typing.Tuple[str, typing.List[ return (prefix, remainders) def list_k32_plots(d: str) -> typing.List[str]: - 'List completed k32 plots in a directory (not recursive)' + 'List completed k32 (and others) plots in a directory (not recursive)' plots = [] for plot in os.listdir(d): - if re.match(r'^plot-k32-.*plot$', plot): + if matches := re.search(r"^plot-k(\d*)-.*plot$", plot): + grps = matches.groups() + plot_k = int(grps[0]) plot = os.path.join(d, plot) try: - if os.stat(plot).st_size > (0.95 * get_k32_plotsize()): + if os.stat(plot).st_size > (0.95 * get_plotsize(plot_k)): plots.append(plot) except FileNotFoundError: continue - return plots def column_wrap( From fc3996ac3b6a70fd993aa1acf81c5961c08deae7 Mon Sep 17 00:00:00 2001 From: Pablo Garcia de los Salmones Valencia Date: Wed, 23 Jun 2021 02:02:42 +0200 Subject: [PATCH 070/109] Generalized code form k32 to any size of k --- src/plotman/_tests/plot_util_test.py | 4 ++-- src/plotman/archive.py | 4 ++-- src/plotman/plot_util.py | 7 ++----- src/plotman/reporting.py | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/plotman/_tests/plot_util_test.py b/src/plotman/_tests/plot_util_test.py index 4a9a8c0b..5a0f9879 100755 --- a/src/plotman/_tests/plot_util_test.py +++ b/src/plotman/_tests/plot_util_test.py @@ -45,7 +45,7 @@ def test_columns() -> None: [ 1 ], [ 2 ] ] ) -def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: +def test_list_k_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: fs.create_file('/t/plot-k32-0.plot', st_size=108 * GB) fs.create_file('/t/plot-k32-1.plot', st_size=108 * GB) fs.create_file('/t/.plot-k32-2.plot', st_size=108 * GB) @@ -53,7 +53,7 @@ def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: fs.create_file('/t/plot-k32-4.plot', st_size=100 * GB) fs.create_file('/t/plot-k32-5.plot', st_size=108 * GB) - assert (plot_util.list_k32_plots('/t/') == + assert (plot_util.list_k_plots('/t/') == [ '/t/plot-k32-0.plot', '/t/plot-k32-1.plot', '/t/plot-k32-5.plot' ] ) diff --git a/src/plotman/archive.py b/src/plotman/archive.py index 13c36ddd..0c78ee07 100644 --- a/src/plotman/archive.py +++ b/src/plotman/archive.py @@ -96,7 +96,7 @@ def spawn_archive_process(dir_cfg: configuration.Directories, arch_cfg: configur def compute_priority(phase: job.Phase, gb_free: float, n_plots: int) -> int: # All these values are designed around dst buffer dirs of about - # ~2TB size and containing k32 plots. TODO: Generalize, and + # ~2TB size and containing k plots. TODO: Generalize, and # rewrite as a sort function. priority = 50 @@ -210,7 +210,7 @@ def archive(dir_cfg: configuration.Directories, arch_cfg: configuration.Archivin dst_dir = dir_cfg.get_dst_directories() for d in dst_dir: ph = dir2ph.get(d, job.Phase(0, 0)) - dir_plots = plot_util.list_k32_plots(d) + dir_plots = plot_util.list_k_plots(d) gb_free = plot_util.df_b(d) / plot_util.GB n_plots = len(dir_plots) priority = compute_priority(ph, gb_free, n_plots) diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py index 5238f5e4..5bad17bd 100644 --- a/src/plotman/plot_util.py +++ b/src/plotman/plot_util.py @@ -14,9 +14,6 @@ def df_b(d: str) -> int: usage = shutil.disk_usage(d) return usage.free -def get_k32_plotsize() -> int: - return get_plotsize(32) - def get_plotsize(k: int) -> int: return (int)(_get_plotsize_scaler(k) * k * pow(2, k)) @@ -54,8 +51,8 @@ def split_path_prefix(items: typing.List[str]) -> typing.Tuple[str, typing.List[ remainders = [ os.path.relpath(i, prefix) for i in items ] return (prefix, remainders) -def list_k32_plots(d: str) -> typing.List[str]: - 'List completed k32 (and others) plots in a directory (not recursive)' +def list_k_plots(d: str) -> typing.List[str]: + 'List completed plots in a directory (not recursive)' plots = [] for plot in os.listdir(d): if matches := re.search(r"^plot-k(\d*)-.*plot$", plot): diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 0a74d84d..be2fca9f 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -215,7 +215,7 @@ def dst_dir_report(jobs: typing.List[job.Job], dstdirs: typing.List[str], width: eldest_ph = dir2oldphase.get(d, job.Phase(0, 0)) phases = job.job_phases_for_dstdir(d, jobs) - dir_plots = plot_util.list_k32_plots(d) + dir_plots = plot_util.list_k_plots(d) gb_free = int(plot_util.df_b(d) / plot_util.GB) n_plots = len(dir_plots) priority = archive.compute_priority(eldest_ph, gb_free, n_plots) From e62f67fe90ab0a2ab643ec229cb828c7cf728579 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 18:22:32 -0700 Subject: [PATCH 071/109] Apply suggestions from code review --- src/plotman/job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 4c60998c..3009df6b 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -268,10 +268,10 @@ def __init__( # 'exclude_final_dir': False, # } if proc.name().startswith("chia_plot"): # MADMAX - self.k = 32 # type: ignore[assignment] + self.k = 32 self.r = self.args['threads'] # type: ignore[assignment] self.u = self.args['buckets'] # type: ignore[assignment] - self.b = 0 # type: ignore[assignment] + self.b = 0 self.n = self.args['count'] # type: ignore[assignment] self.tmpdir = self.args['tmpdir'] # type: ignore[assignment] self.tmp2dir = self.args['tmpdir2'] # type: ignore[assignment] From 758073c7e6bb216dfbb6f884dfc81969026baeb8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 21:37:41 -0400 Subject: [PATCH 072/109] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd4af43b..8f15f809 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#430](https://github.com/ericaltendorf/plotman/pull/430)) - `plotman logs` command to print and tail plot logs by their plot ID. ([#509](https://github.com/ericaltendorf/plotman/pull/509)) -- Initial support for MadMax plotter. +- Support the [MadMax plotter](https://github.com/madMAx43v3r/chia-plotter). + See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. ([#797](https://github.com/ericaltendorf/plotman/pull/797)) ## [0.4.1] - 2021-06-11 From 53587e314010f9ff13b317135d54c2ca01ca7a8c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 22:11:08 -0400 Subject: [PATCH 073/109] Update plotman.yaml --- src/plotman/resources/plotman.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 49042676..b33c5fdd 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -136,7 +136,7 @@ scheduling: # See documentation at # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create plotting: - # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt + # Your public keys: farmer and pool - Required for madMAx, optional for chia with mnemonic.txt # farmer_pk: ... # pool_pk: ... @@ -150,8 +150,8 @@ plotting: n_buckets: 128 # Number of buckets to split data into job_buffer: 3389 # Per job memory - # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs - madmax: - # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter + # If you enable madMAx, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs + madMAx: + # madMAx plotter: https://github.com/madMAx43v3r/chia-plotter n_threads: 4 # Default is 4, crank up if you have many cores n_buckets: 256 # Default is 256 From f539c73f530392050c9518ecc8a82d38fd10bed1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 22:11:57 -0400 Subject: [PATCH 074/109] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f15f809..4e717de8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#430](https://github.com/ericaltendorf/plotman/pull/430)) - `plotman logs` command to print and tail plot logs by their plot ID. ([#509](https://github.com/ericaltendorf/plotman/pull/509)) -- Support the [MadMax plotter](https://github.com/madMAx43v3r/chia-plotter). +- Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter). See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. ([#797](https://github.com/ericaltendorf/plotman/pull/797)) From b788def591159f5345ffe8da6dc99fc7abbd49a4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 22:19:30 -0400 Subject: [PATCH 075/109] Update plotman.yaml --- src/plotman/resources/plotman.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index b33c5fdd..1ccfd9c4 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -151,7 +151,7 @@ plotting: job_buffer: 3389 # Per job memory # If you enable madMAx, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs - madMAx: + madmax: # madMAx plotter: https://github.com/madMAx43v3r/chia-plotter n_threads: 4 # Default is 4, crank up if you have many cores n_buckets: 256 # Default is 256 From a8f24b5242c1b4dd68205214b00a38f084db1c82 Mon Sep 17 00:00:00 2001 From: Rafael Steil Date: Mon, 21 Jun 2021 11:53:15 -0400 Subject: [PATCH 076/109] Look from job temp files in the filesystem instead of open files --- src/plotman/job.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 3009df6b..c99ae257 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -7,6 +7,7 @@ import random import re import sys +import glob import time from datetime import datetime from enum import Enum, auto @@ -555,13 +556,11 @@ def resume(self) -> None: def get_temp_files(self) -> typing.Set[str]: # Prevent duplicate file paths by using set. temp_files = set([]) - for f in self.proc.open_files(): - if any( - dir in f.path - for dir in [self.tmpdir, self.tmp2dir, self.dstdir] - if dir is not None - ): - temp_files.add(f.path) + + for dir in [self.tmpdir, self.tmp2dir, self.dstdir]: + if dir is not None: + temp_files.update(glob.glob(os.path.join(dir, "plot-*-{0}.*".format(self.plot_id)))) + return temp_files def cancel(self) -> None: From 834e8240830416a8c84b645a5f1e8089dfae554c Mon Sep 17 00:00:00 2001 From: Rafael Steil Date: Mon, 21 Jun 2021 11:34:15 -0400 Subject: [PATCH 077/109] Add -f argument to plotman kill to bypass confirmation --- src/plotman/job.py | 2 +- src/plotman/plotman.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index c99ae257..96dd72fb 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -559,7 +559,7 @@ def get_temp_files(self) -> typing.Set[str]: for dir in [self.tmpdir, self.tmp2dir, self.dstdir]: if dir is not None: - temp_files.update(glob.glob(os.path.join(dir, "plot-*-{0}.*".format(self.plot_id)))) + temp_files.update(glob.glob(os.path.join(dir, f"plot-*-{self.plot_id}.*"))) return temp_files diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 7aa9536c..3b67ef6a 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -73,6 +73,7 @@ def parse_args(self) -> typing.Any: self.add_idprefix_arg(p_files) p_kill = sp.add_parser('kill', help='kill job (and cleanup temp files)') + p_kill.add_argument('-f', '--force', action='store_true', default=False, help="Don't ask for confirmation before killing the plot job") self.add_idprefix_arg(p_kill) p_suspend = sp.add_parser('suspend', help='suspend job') @@ -309,15 +310,23 @@ def main() -> None: job.suspend() temp_files = job.get_temp_files() - print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id)) - print('Will delete %d temp files' % len(temp_files)) - conf = input('Are you sure? ("y" to confirm): ') + + if args.force: + conf = 'y' + else: + conf = input('Are you sure? ("y" to confirm): ') + if (conf != 'y'): - print('canceled. If you wish to resume the job, do so manually.') + print('Canceled. If you wish to resume the job, do so manually.') else: + print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id)) + print('Will delete %d temp files' % len(temp_files)) print('killing...') + job.cancel() + print('cleaning up temp files...') + for f in temp_files: os.remove(f) From 49f97f0d93ea61d398f9143e4d82a09f5b0c5262 Mon Sep 17 00:00:00 2001 From: Rafael Steil Date: Tue, 22 Jun 2021 23:46:04 -0400 Subject: [PATCH 078/109] Update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e717de8..6db63086 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter). See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. ([#797](https://github.com/ericaltendorf/plotman/pull/797)) +- Added argument `-f / --force` to `plotman kill` to skip confirmation before killing the job + +### Fixed +- `plotman kill` doesn't leave any temporary files behind anymore ## [0.4.1] - 2021-06-11 ### Fixed From d80aa87dcd52e541fb652257dae47b441140be4b Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Wed, 23 Jun 2021 12:33:24 +0100 Subject: [PATCH 079/109] Add uid/gid args --- build-docker-plotman.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh index 61809a11..1f3e0bc4 100755 --- a/build-docker-plotman.sh +++ b/build-docker-plotman.sh @@ -6,6 +6,8 @@ PROJECT="chia-plotman" TAG="plotter" BASE_CONTAINER="ubuntu:20.04" CHIA_GIT_REFERENCE="1.1.7" +UID=10001 +GID=10001 docker rmi ${LOCAL_REGISTRY}/${PROJECT}:${TAG} @@ -13,6 +15,8 @@ docker build . \ --squash \ --build-arg BASE_CONTAINER=${BASE_CONTAINER} \ --build-arg CHIA_GIT_REFERENCE=${CHIA_GIT_REFERENCE} \ + --build-arg UID=${UID} \ + --build-arg GID=${GID} \ -f Dockerfile \ -t ${LOCAL_REGISTRY}/${PROJECT}:${TAG} # -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} From 336d73c9acf67fcd35d47ecea28810e76968d1e7 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Wed, 23 Jun 2021 13:21:51 +0100 Subject: [PATCH 080/109] Move docker artifacts to own subdirectory --- .dockerfile => .dockerignore | 1 + build-docker-plotman.sh | 6 ++++-- Dockerfile => docker/Dockerfile | 0 3 files changed, 5 insertions(+), 2 deletions(-) rename .dockerfile => .dockerignore (83%) rename Dockerfile => docker/Dockerfile (100%) diff --git a/.dockerfile b/.dockerignore similarity index 83% rename from .dockerfile rename to .dockerignore index 756b8274..aba6e3ad 100644 --- a/.dockerfile +++ b/.dockerignore @@ -2,3 +2,4 @@ .github .coveragerc .gitignore +docker diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh index 1f3e0bc4..24a23698 100755 --- a/build-docker-plotman.sh +++ b/build-docker-plotman.sh @@ -6,6 +6,8 @@ PROJECT="chia-plotman" TAG="plotter" BASE_CONTAINER="ubuntu:20.04" CHIA_GIT_REFERENCE="1.1.7" + +# The UID/GID should match the 'chia' owner of the directories on the host system UID=10001 GID=10001 @@ -17,9 +19,9 @@ docker build . \ --build-arg CHIA_GIT_REFERENCE=${CHIA_GIT_REFERENCE} \ --build-arg UID=${UID} \ --build-arg GID=${GID} \ - -f Dockerfile \ + -f docker/Dockerfile \ -t ${LOCAL_REGISTRY}/${PROJECT}:${TAG} -# -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} +# -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} docker push ${LOCAL_REGISTRY}/${PROJECT}:${TAG} #docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG} diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile From 70f281f0a655e62c7d1c4c76c51a5b423f982738 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Wed, 23 Jun 2021 13:46:32 +0100 Subject: [PATCH 081/109] Tidy up registry reference --- build-docker-plotman.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh index 24a23698..1ae200b0 100755 --- a/build-docker-plotman.sh +++ b/build-docker-plotman.sh @@ -1,7 +1,6 @@ #!/bin/bash -LOCAL_REGISTRY="" -#DOCKER_REGISTRY="" +DOCKER_REGISTRY="" PROJECT="chia-plotman" TAG="plotter" BASE_CONTAINER="ubuntu:20.04" @@ -20,8 +19,6 @@ docker build . \ --build-arg UID=${UID} \ --build-arg GID=${GID} \ -f docker/Dockerfile \ - -t ${LOCAL_REGISTRY}/${PROJECT}:${TAG} -# -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} + -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} -docker push ${LOCAL_REGISTRY}/${PROJECT}:${TAG} -#docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG} +docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG} From 46cb570d2881ee545062c649bcea9daeae02a0da Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Wed, 23 Jun 2021 19:13:43 +0100 Subject: [PATCH 082/109] Add samples and update CHANGELOG --- CHANGELOG.md | 4 ++++ docker/sample.docker-compose.yml | 19 +++++++++++++++++++ docker/sample.env | 7 +++++++ 3 files changed, 30 insertions(+) create mode 100644 docker/sample.docker-compose.yml create mode 100644 docker/sample.env diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e717de8..26481fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter). See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. ([#797](https://github.com/ericaltendorf/plotman/pull/797)) +- Docker container support. + See the [docker configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Docker-Configuration) for help setting it up. + ([#783](https://github.com/ericaltendorf/plotman/pull/783)) + ## [0.4.1] - 2021-06-11 ### Fixed diff --git a/docker/sample.docker-compose.yml b/docker/sample.docker-compose.yml new file mode 100644 index 00000000..af6ae14f --- /dev/null +++ b/docker/sample.docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" + +services: + chia_plotman: + restart: always + container_name: chia-plotman + image: ${DOCKER_IMAGE} + volumes: + - ${HOME}/.ssh:/home/chia/.ssh + - ${HOME}/.chia:/home/chia/.chia + - ${HOME}/.config:/home/chia/.config + - ${LOGS_DIR}:/data/chia/logs + - ${PLOTS_DIR}:/data/chia/plots + - ${PLOTS_TMP_DIR}:/data/chia/tmp + - /tmp:/tmp + logging: + options: + max-size: ${DOCKER_LOG_MAX_SIZE} + max-file: ${DOCKER_LOG_MAX_FILE} diff --git a/docker/sample.env b/docker/sample.env new file mode 100644 index 00000000..725fd840 --- /dev/null +++ b/docker/sample.env @@ -0,0 +1,7 @@ +DOCKER_IMAGE=/chia-plotman:plotter +LOGS_DIR=/data/chia/logs +PLOTS_DIR=/data/chia/plots +PLOTS_TMP_DIR=/data/chia/tmp +DOCKER_LOG_MAX_SIZE=4m +DOCKER_LOG_MAX_FILE=10 + From 1429ef8b52397a8000f7d8ed123486800723c76f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 24 Jun 2021 19:37:23 -0700 Subject: [PATCH 083/109] Apply suggestions from code review --- CHANGELOG.md | 2 +- src/plotman/job.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6db63086..f6330aea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter). See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. ([#797](https://github.com/ericaltendorf/plotman/pull/797)) -- Added argument `-f / --force` to `plotman kill` to skip confirmation before killing the job +- Added argument `-f`/`--force` to `plotman kill` to skip confirmation before killing the job. ### Fixed - `plotman kill` doesn't leave any temporary files behind anymore diff --git a/src/plotman/job.py b/src/plotman/job.py index 96dd72fb..ffa0dc71 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -559,7 +559,7 @@ def get_temp_files(self) -> typing.Set[str]: for dir in [self.tmpdir, self.tmp2dir, self.dstdir]: if dir is not None: - temp_files.update(glob.glob(os.path.join(dir, f"plot-*-{self.plot_id}.*"))) + temp_files.update(glob.glob(os.path.join(dir, f"plot-*-{self.plot_id}.tmp"))) return temp_files From 5adea47069790356bd3c0631cf88a6c8903b4af6 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 24 Jun 2021 22:39:39 -0400 Subject: [PATCH 084/109] Update CHANGELOG.md --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6330aea..48102953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [unreleased] +### Fixed +- `plotman kill` doesn't leave any temporary files behind anymore. + ([#801](https://github.com/ericaltendorf/plotman/pull/801)) ### Added - `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) @@ -21,9 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. ([#797](https://github.com/ericaltendorf/plotman/pull/797)) - Added argument `-f`/`--force` to `plotman kill` to skip confirmation before killing the job. - -### Fixed -- `plotman kill` doesn't leave any temporary files behind anymore + ([#801](https://github.com/ericaltendorf/plotman/pull/801)) ## [0.4.1] - 2021-06-11 ### Fixed From 078f353ba5b3b0b45e74c15b640a31dfba951308 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 24 Jun 2021 22:41:07 -0400 Subject: [PATCH 085/109] Update plotman.py --- src/plotman/plotman.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 3b67ef6a..5312b78c 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -311,6 +311,9 @@ def main() -> None: temp_files = job.get_temp_files() + print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id)) + print('Will delete %d temp files' % len(temp_files)) + if args.force: conf = 'y' else: @@ -319,8 +322,6 @@ def main() -> None: if (conf != 'y'): print('Canceled. If you wish to resume the job, do so manually.') else: - print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id)) - print('Will delete %d temp files' % len(temp_files)) print('killing...') job.cancel() From 1e20fce67130f5c9f785944acd584223ed3437a9 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 27 Jun 2021 16:59:01 -0400 Subject: [PATCH 086/109] update MANIFEST.in for Docker files --- MANIFEST.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 08d64bbe..8e5ab768 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,3 +9,6 @@ include .coveragerc recursive-include src *.py recursive-include src/plotman/_tests/resources * recursive-include src/plotman/resources * +recursive-exclude docker * +exclude .dockerignore +exclude build-docker-plotman.sh From eb82eaa9631e78f50781f842fbf3df008641894c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 28 Jun 2021 22:42:56 -0400 Subject: [PATCH 087/109] some references are still k32 --- src/plotman/_tests/plot_util_test.py | 4 ++-- src/plotman/archive.py | 4 ++-- src/plotman/plot_util.py | 2 +- src/plotman/reporting.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/plotman/_tests/plot_util_test.py b/src/plotman/_tests/plot_util_test.py index 5a0f9879..ab912e55 100755 --- a/src/plotman/_tests/plot_util_test.py +++ b/src/plotman/_tests/plot_util_test.py @@ -45,7 +45,7 @@ def test_columns() -> None: [ 1 ], [ 2 ] ] ) -def test_list_k_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: +def test_list_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: fs.create_file('/t/plot-k32-0.plot', st_size=108 * GB) fs.create_file('/t/plot-k32-1.plot', st_size=108 * GB) fs.create_file('/t/.plot-k32-2.plot', st_size=108 * GB) @@ -53,7 +53,7 @@ def test_list_k_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: fs.create_file('/t/plot-k32-4.plot', st_size=100 * GB) fs.create_file('/t/plot-k32-5.plot', st_size=108 * GB) - assert (plot_util.list_k_plots('/t/') == + assert (plot_util.list_plots('/t/') == [ '/t/plot-k32-0.plot', '/t/plot-k32-1.plot', '/t/plot-k32-5.plot' ] ) diff --git a/src/plotman/archive.py b/src/plotman/archive.py index 0c78ee07..59a9a9e6 100644 --- a/src/plotman/archive.py +++ b/src/plotman/archive.py @@ -96,7 +96,7 @@ def spawn_archive_process(dir_cfg: configuration.Directories, arch_cfg: configur def compute_priority(phase: job.Phase, gb_free: float, n_plots: int) -> int: # All these values are designed around dst buffer dirs of about - # ~2TB size and containing k plots. TODO: Generalize, and + # ~2TB size and containing k32 plots. TODO: Generalize, and # rewrite as a sort function. priority = 50 @@ -210,7 +210,7 @@ def archive(dir_cfg: configuration.Directories, arch_cfg: configuration.Archivin dst_dir = dir_cfg.get_dst_directories() for d in dst_dir: ph = dir2ph.get(d, job.Phase(0, 0)) - dir_plots = plot_util.list_k_plots(d) + dir_plots = plot_util.list_plots(d) gb_free = plot_util.df_b(d) / plot_util.GB n_plots = len(dir_plots) priority = compute_priority(ph, gb_free, n_plots) diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py index 5bad17bd..38d5f78c 100644 --- a/src/plotman/plot_util.py +++ b/src/plotman/plot_util.py @@ -51,7 +51,7 @@ def split_path_prefix(items: typing.List[str]) -> typing.Tuple[str, typing.List[ remainders = [ os.path.relpath(i, prefix) for i in items ] return (prefix, remainders) -def list_k_plots(d: str) -> typing.List[str]: +def list_plots(d: str) -> typing.List[str]: 'List completed plots in a directory (not recursive)' plots = [] for plot in os.listdir(d): diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 83fe5230..75ad5a8c 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -216,7 +216,7 @@ def dst_dir_report(jobs: typing.List[job.Job], dstdirs: typing.List[str], width: eldest_ph = dir2oldphase.get(d, job.Phase(0, 0)) phases = job.job_phases_for_dstdir(d, jobs) - dir_plots = plot_util.list_k_plots(d) + dir_plots = plot_util.list_plots(d) gb_free = int(plot_util.df_b(d) / plot_util.GB) n_plots = len(dir_plots) priority = archive.compute_priority(eldest_ph, gb_free, n_plots) From 20284713d6475100519db4656e6d042d66b53128 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 28 Jun 2021 22:43:04 -0400 Subject: [PATCH 088/109] add changelog entry for #803 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf67106..adb3cdbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Docker container support. See the [docker configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Docker-Configuration) for help setting it up. ([#783](https://github.com/ericaltendorf/plotman/pull/783)) +- Plot sizes other than k32 are handled. + ([#803](https://github.com/ericaltendorf/plotman/pull/803)) ## [0.4.1] - 2021-06-11 ### Fixed From deace18001c0a471c3cdfefb5108f2b87c2c85cb Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 28 Jun 2021 22:45:43 -0400 Subject: [PATCH 089/109] add a small k33 test case --- src/plotman/_tests/plot_util_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plotman/_tests/plot_util_test.py b/src/plotman/_tests/plot_util_test.py index ab912e55..6718cb44 100755 --- a/src/plotman/_tests/plot_util_test.py +++ b/src/plotman/_tests/plot_util_test.py @@ -53,10 +53,14 @@ def test_list_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: fs.create_file('/t/plot-k32-4.plot', st_size=100 * GB) fs.create_file('/t/plot-k32-5.plot', st_size=108 * GB) + fs.create_file('/t/plot-k33-6.plot', st_size=108 * GB) + fs.create_file('/t/plot-k33-7.plot', st_size=216 * GB) + assert (plot_util.list_plots('/t/') == [ '/t/plot-k32-0.plot', '/t/plot-k32-1.plot', - '/t/plot-k32-5.plot' ] ) + '/t/plot-k32-5.plot', + '/t/plot-k33-7.plot' ] ) def test_get_plotsize() -> None: From b4f8b905281672243ecace2e7f409caf6f4786b4 Mon Sep 17 00:00:00 2001 From: racergoodwin <84841970+racergoodwin@users.noreply.github.com> Date: Tue, 29 Jun 2021 21:56:06 +0100 Subject: [PATCH 090/109] Update CHANGELOG.md Agree to commit suggestion Co-authored-by: Kyle Altendorf --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e920bae..40dac73f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `plotman kill` doesn't leave any temporary files behind anymore. ([#801](https://github.com/ericaltendorf/plotman/pull/801)) ### Added -- optional tmpdir_overrides added for each specified tmpdir +- tmp directory overrides moved to `scheduling:` `tmp_overrides:`. + ([#758](https://github.com/ericaltendorf/plotman/pull/758)) +- Per tmp directory phase limit control added to `scheduling:` `tmp_overrides:`. ([#758](https://github.com/ericaltendorf/plotman/pull/758)) - `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) From c7678e28d14283f20be3e108d1b824243ad628ff Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 29 Jun 2021 19:06:12 -0400 Subject: [PATCH 091/109] check chia version is sufficient for pools --- setup.cfg | 1 + src/plotman/configuration.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/setup.cfg b/setup.cfg index f922b21e..739c8161 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ install_requires = click ~= 7.1 desert == 2020.11.18 marshmallow ~= 3.12 + packaging ~= 20.9 pendulum ~= 2.1 psutil ~= 5.8 pyyaml ~= 5.4 diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 6d59b936..50bf4703 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -2,6 +2,7 @@ import importlib import os import stat +import subprocess import tempfile import textwrap from typing import Dict, Generator, List, Mapping, Optional @@ -14,6 +15,7 @@ import marshmallow import marshmallow.fields import marshmallow.validate +import packaging.utils import pendulum import yaml @@ -354,6 +356,22 @@ class PlotmanConfig: @contextlib.contextmanager def setup(self) -> Generator[None, None, None]: + if self.plotting.type == 'chia': + if self.plotting.chia.pool_contract_address is not None: + completed_process = subprocess.run( + args=['chia', 'version'], + capture_output=True, + check=True, + encoding='utf-8', + ) + version = packaging.utils.Version(completed_process.stdout) + required_version = packaging.utils.Version('1.2') + if version < required_version: + raise Exception( + f'Chia version {required_version} required for creating pool' + f' plots but found: {version}' + ) + prefix = f'plotman-pid_{os.getpid()}-' self.logging.setup() From c09e8c0a5aeab9b5532916bd4b49b8b66e33059e Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 29 Jun 2021 19:29:53 -0400 Subject: [PATCH 092/109] mypy --- src/plotman/configuration.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 50bf4703..ea57b9bc 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -15,7 +15,7 @@ import marshmallow import marshmallow.fields import marshmallow.validate -import packaging.utils +import packaging.version import pendulum import yaml @@ -356,7 +356,7 @@ class PlotmanConfig: @contextlib.contextmanager def setup(self) -> Generator[None, None, None]: - if self.plotting.type == 'chia': + if self.plotting.type == 'chia' and self.plotting.chia is not None: if self.plotting.chia.pool_contract_address is not None: completed_process = subprocess.run( args=['chia', 'version'], @@ -364,8 +364,8 @@ def setup(self) -> Generator[None, None, None]: check=True, encoding='utf-8', ) - version = packaging.utils.Version(completed_process.stdout) - required_version = packaging.utils.Version('1.2') + version = packaging.version.Version(completed_process.stdout) + required_version = packaging.version.Version('1.2') if version < required_version: raise Exception( f'Chia version {required_version} required for creating pool' From 1a4a055ea8d3fbb848c11bd1044c052c22a23502 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 29 Jun 2021 20:09:51 -0400 Subject: [PATCH 093/109] no walrus in 3.7 --- src/plotman/plot_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py index 38d5f78c..75d85eca 100644 --- a/src/plotman/plot_util.py +++ b/src/plotman/plot_util.py @@ -55,7 +55,8 @@ def list_plots(d: str) -> typing.List[str]: 'List completed plots in a directory (not recursive)' plots = [] for plot in os.listdir(d): - if matches := re.search(r"^plot-k(\d*)-.*plot$", plot): + matches = re.search(r"^plot-k(\d*)-.*plot$", plot) + if matches is not None: grps = matches.groups() plot_k = int(grps[0]) plot = os.path.join(d, plot) From a4ef41f4cbefb35ecc203e5680163aebc81fad0e Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 29 Jun 2021 22:28:12 -0400 Subject: [PATCH 094/109] make it a raw string --- src/plotman/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index ffa0dc71..6fa2b410 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -333,7 +333,7 @@ def init_from_logfile(self) -> None: self.plotter = 'chia' found_id = True else: - m = re.match("^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line) + m = re.match(r"^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line) if m: # MADMAX self.plot_id = m.group(7) self.plotter = 'madmax' From 18e2fcc9c7d91eb5a02924a01120580e7438a3a9 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 30 Jun 2021 09:14:50 -0400 Subject: [PATCH 095/109] fix job-missing row creation --- src/plotman/reporting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 75ad5a8c..46ea8786 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -113,7 +113,8 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona ] except (psutil.NoSuchProcess, psutil.AccessDenied): # In case the job has disappeared - row = [j.plot_id[:8]] + (['--'] * 12) + row = ['--' for _ in headings] + row[0] = j.plot_id[:8] if height: row.insert(0, '%3d' % i) From 6dbd78b09d44ac76127b646acb67062df16de1a4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 30 Jun 2021 09:41:52 -0400 Subject: [PATCH 096/109] Update reporting.py --- src/plotman/reporting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 46ea8786..2eccb9ee 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -113,8 +113,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona ] except (psutil.NoSuchProcess, psutil.AccessDenied): # In case the job has disappeared - row = ['--' for _ in headings] - row[0] = j.plot_id[:8] + row = [j.plot_id[:8]] + (['--'] * (len(headings) - 1)) if height: row.insert(0, '%3d' % i) From 7145c04f7de751a7a22e369a02fd449d08333b30 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Wed, 30 Jun 2021 19:24:52 +0100 Subject: [PATCH 097/109] Only run regex for configured plotter type --- src/plotman/job.py | 91 +++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 6fa2b410..9de35a6b 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -19,7 +19,7 @@ import pendulum import psutil -from plotman import chia, madmax +from plotman import configuration, chia, madmax def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: @@ -378,51 +378,52 @@ def set_phase_from_logfile(self) -> None: with open(self.logfile, 'r') as f: with contextlib.suppress(UnicodeDecodeError): for line in f: - # CHIA: "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" - m = re.match(r'^Starting phase (\d).*', line) - if m: - phase = int(m.group(1)) - phase_subphases[phase] = 0 - - # MADMAX: "[P1]" or "[P2]" or "[P3]" or "[P4]" - m = re.match(r'^\[P(\d)\].*', line) - if m: - phase = int(m.group(1)) - phase_subphases[phase] = 0 - - # CHIA: Phase 1: "Computing table 2" - m = re.match(r'^Computing table (\d).*', line) - if m: - phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) - - # MADMAX: Phase 1: "[P1] Table 2" - m = re.match(r'^\[P1\] Table (\d).*', line) - if m: - phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) - - # CHIA: Phase 2: "Backpropagating on table 2" - m = re.match(r'^Backpropagating on table (\d).*', line) - if m: - phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) - - # MADMAX: Phase 2: "[P2] Table 2" - m = re.match(r'^\[P2\] Table (\d).*', line) - if m: - phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) - - # CHIA: Phase 3: "Compressing tables 4 and 5" - m = re.match(r'^Compressing tables (\d) and (\d).*', line) - if m: - phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) - - # MADMAX: Phase 3: "[P3-1] Table 4" - m = re.match(r'^\[P3\-\d\] Table (\d).*', line) - if m: - if 3 in phase_subphases: + if configuration.Plotting.type == "madmax": + # MADMAX: "[P1]" or "[P2]" or "[P3]" or "[P4]" + m = re.match(r'^\[P(\d)\].*', line) + if m: + phase = int(m.group(1)) + phase_subphases[phase] = 0 + + # MADMAX: Phase 1: "[P1] Table 2" + m = re.match(r'^\[P1\] Table (\d).*', line) + if m: + phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) + + # MADMAX: Phase 2: "[P2] Table 2" + m = re.match(r'^\[P2\] Table (\d).*', line) + if m: + phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) + + # MADMAX: Phase 3: "[P3-1] Table 4" + m = re.match(r'^\[P3\-\d\] Table (\d).*', line) + if m: + if 3 in phase_subphases: + phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + else: + phase_subphases[3] = int(m.group(1)) + else: + # CHIA: "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" + m = re.match(r'^Starting phase (\d).*', line) + if m: + phase = int(m.group(1)) + phase_subphases[phase] = 0 + + # CHIA: Phase 1: "Computing table 2" + m = re.match(r'^Computing table (\d).*', line) + if m: + phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) + + # CHIA: Phase 2: "Backpropagating on table 2" + m = re.match(r'^Backpropagating on table (\d).*', line) + if m: + phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) + + # CHIA: Phase 3: "Compressing tables 4 and 5" + m = re.match(r'^Compressing tables (\d) and (\d).*', line) + if m: phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) - else: - phase_subphases[3] = int(m.group(1)) - + # TODO also collect timing info: # "Time for phase 1 = 22796.7 seconds. CPU (98%) Tue Sep 29 17:57:19 2020" From 8d30f6cc14efd7fe258fff75901a55b53925ea8b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 30 Jun 2021 21:26:17 -0400 Subject: [PATCH 098/109] normpath for some directory comparisons --- src/plotman/job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 6fa2b410..c7f52f55 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -24,11 +24,11 @@ def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: '''Return phase 2-tuples for jobs running on tmpdir d''' - return sorted([j.progress() for j in all_jobs if j.tmpdir == d]) + return sorted([j.progress() for j in all_jobs if os.path.normpath(j.tmpdir) == os.path.normpath(d)]) def job_phases_for_dstdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: '''Return phase 2-tuples for jobs outputting to dstdir d''' - return sorted([j.progress() for j in all_jobs if j.dstdir == d]) + return sorted([j.progress() for j in all_jobs if os.path.normpath(j.dstdir) == os.path.normpath(d)]) def is_plotting_cmdline(cmdline: typing.List[str]) -> bool: if cmdline and 'python' in cmdline[0].lower(): # Stock Chia plotter From dba58b7f41e6c7eaa55dfaf8ec66094cc22b57d6 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 30 Jun 2021 22:18:26 -0400 Subject: [PATCH 099/109] add pool contract address support for madMAx --- src/plotman/configuration.py | 46 ++++++++++++++++++++++++++---------- src/plotman/manager.py | 7 ++++-- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 3de8ddb3..d1fbb1c5 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -71,26 +71,35 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi f"Config file at: '{config_path}' is malformed" ) from e - if loaded.plotting.type == "chia" and loaded.plotting.chia is None: - raise ConfigurationException( - "chia selected as plotter but plotting: chia: was not specified in the config", - ) + if loaded.plotting.type == "chia": + if loaded.plotting.chia is None: + raise ConfigurationException( + "chia selected as plotter but plotting: chia: was not specified in the config", + ) + + if loaded.plotting.pool_pk is not None and loaded.plotting.pool_contract_address is not None: + raise ConfigurationException( + "Chia Network plotter accepts up to one of plotting: pool_pk: and pool_contract_address: but both are specified", + ) elif loaded.plotting.type == "madmax": if loaded.plotting.madmax is None: raise ConfigurationException( - "madmax selected as plotter but plotting: madmax: was not specified in the config", + "madMAx selected as plotter but plotting: madmax: was not specified in the config", ) if loaded.plotting.farmer_pk is None: raise ConfigurationException( - "madmax selected as plotter but no plotting: farmer_pk: was specified in the config", + "madMAx selected as plotter but no plotting: farmer_pk: was specified in the config", ) - if loaded.plotting.pool_pk is None: + if loaded.plotting.pool_pk is None and loaded.plotting.pool_contract_address is None: raise ConfigurationException( - "madmax selected as plotter but no plotting: pool_pk: was specified in the config", + "madMAx plotter requires one of plotting: pool_pk: or pool_contract_address: to be specified but neither is", + ) + elif loaded.plotting.pool_pk is not None and loaded.plotting.pool_contract_address is not None: + raise ConfigurationException( + "madMAx plotter accepts only one of plotting: pool_pk: and pool_contract_address: but both are specified", ) - if loaded.archiving is not None: preset_target_objects = yaml.safe_load(preset_target_definitions_text) @@ -309,7 +318,6 @@ class ChiaPlotterOptions: e: Optional[bool] = False job_buffer: Optional[int] = 3389 x: bool = False - pool_contract_address: Optional[str] = None @attr.frozen class MadmaxPlotterOptions: @@ -320,6 +328,7 @@ class MadmaxPlotterOptions: class Plotting: farmer_pk: Optional[str] = None pool_pk: Optional[str] = None + pool_contract_address: Optional[str] = None type: str = attr.ib( default="chia", metadata={ @@ -359,8 +368,8 @@ class PlotmanConfig: @contextlib.contextmanager def setup(self) -> Generator[None, None, None]: - if self.plotting.type == 'chia' and self.plotting.chia is not None: - if self.plotting.chia.pool_contract_address is not None: + if self.plotting.type == 'chia': + if self.plotting.pool_contract_address is not None: completed_process = subprocess.run( args=['chia', 'version'], capture_output=True, @@ -374,6 +383,19 @@ def setup(self) -> Generator[None, None, None]: f'Chia version {required_version} required for creating pool' f' plots but found: {version}' ) + elif self.plotting.type == 'madmax': + if self.plotting.pool_contract_address is not None: + completed_process = subprocess.run( + args=['chia_plot', '--help'], + capture_output=True, + check=True, + encoding='utf-8', + ) + if '--contract' not in completed_process.stdout: + raise Exception( + f'found madMAx version does not support the `--contract`' + f' option for pools.' + ) prefix = f'plotman-pid_{os.getpid()}-' diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 1565b900..146dc4f7 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -153,6 +153,9 @@ def key(key: str) -> job.Phase: if dir_cfg.tmp2 is not None: plot_args.append('-2') plot_args.append(dir_cfg.tmp2 if dir_cfg.tmp2.endswith('/') else (dir_cfg.tmp2 + '/')) + if plotting_cfg.pool_contract_address is not None: + plot_args.append('-c') + plot_args.append(plotting_cfg.pool_contract_address) else: if plotting_cfg.chia is None: raise Exception( @@ -167,9 +170,9 @@ def key(key: str) -> job.Phase: '-d', dstdir ] if plotting_cfg.chia.e: plot_args.append('-e') - if plotting_cfg.chia.pool_contract_address is not None: + if plotting_cfg.pool_contract_address is not None: plot_args.append('-c') - plot_args.append(plotting_cfg.chia.pool_contract_address) + plot_args.append(plotting_cfg.pool_contract_address) if plotting_cfg.chia.x: plot_args.append('-x') if dir_cfg.tmp2 is not None: From 35686c16b7f73d79483531baf188d6d980252cb2 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 30 Jun 2021 22:51:21 -0400 Subject: [PATCH 100/109] single point for -c --- src/plotman/manager.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 146dc4f7..efe6a2c5 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -153,9 +153,6 @@ def key(key: str) -> job.Phase: if dir_cfg.tmp2 is not None: plot_args.append('-2') plot_args.append(dir_cfg.tmp2 if dir_cfg.tmp2.endswith('/') else (dir_cfg.tmp2 + '/')) - if plotting_cfg.pool_contract_address is not None: - plot_args.append('-c') - plot_args.append(plotting_cfg.pool_contract_address) else: if plotting_cfg.chia is None: raise Exception( @@ -170,9 +167,6 @@ def key(key: str) -> job.Phase: '-d', dstdir ] if plotting_cfg.chia.e: plot_args.append('-e') - if plotting_cfg.pool_contract_address is not None: - plot_args.append('-c') - plot_args.append(plotting_cfg.pool_contract_address) if plotting_cfg.chia.x: plot_args.append('-x') if dir_cfg.tmp2 is not None: @@ -184,7 +178,10 @@ def key(key: str) -> job.Phase: if plotting_cfg.pool_pk is not None: plot_args.append('-p') plot_args.append(plotting_cfg.pool_pk) - + if plotting_cfg.pool_contract_address is not None: + plot_args.append('-c') + plot_args.append(plotting_cfg.pool_contract_address) + logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), log_file_path)) From 1d7704e3215812adc33f43c0c236ad997b70c575 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Thu, 1 Jul 2021 18:10:31 +0100 Subject: [PATCH 101/109] Adjust madmax progress tracking --- src/plotman/job.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 9de35a6b..799ae8b3 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -19,7 +19,7 @@ import pendulum import psutil -from plotman import configuration, chia, madmax +from plotman import chia, madmax def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: @@ -378,30 +378,41 @@ def set_phase_from_logfile(self) -> None: with open(self.logfile, 'r') as f: with contextlib.suppress(UnicodeDecodeError): for line in f: - if configuration.Plotting.type == "madmax": - # MADMAX: "[P1]" or "[P2]" or "[P3]" or "[P4]" + if self.plotter == "madmax": + + # MADMAX reports after completion of phases so increment the reported subphases + # and assume that phase 1 has already started + + # MADMAX: "[P1]" or "[P2]" or "[P4]" m = re.match(r'^\[P(\d)\].*', line) if m: phase = int(m.group(1)) - phase_subphases[phase] = 0 + phase_subphases[phase] = 1 - # MADMAX: Phase 1: "[P1] Table 2" - m = re.match(r'^\[P1\] Table (\d).*', line) + # MADMAX: "[P1] or [P2] Table 7" + m = re.match(r'^\[P(\d)\] Table (\d).*', line) if m: - phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) + phase = int(m.group(1)) + if phase == 1: + phase_subphases[1] = max(phase_subphases[1], (int(m.group(2))+1)) - # MADMAX: Phase 2: "[P2] Table 2" - m = re.match(r'^\[P2\] Table (\d).*', line) - if m: - phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) + elif phase == 2: + if line.find('rewrite') > 0: + phase_subphases[2] = max(phase_subphases[2], (9 - int(m.group(2)))) + else: + phase_subphases[2] = max(phase_subphases[2], (8 - int(m.group(2)))) # MADMAX: Phase 3: "[P3-1] Table 4" - m = re.match(r'^\[P3\-\d\] Table (\d).*', line) + m = re.match(r'^\[P3\-(\d)\] Table (\d).*', line) if m: if 3 in phase_subphases: - phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) - else: - phase_subphases[3] = int(m.group(1)) + if int(m.group(1)) == 2: + phase_subphases[3] = max(phase_subphases[3], int(m.group(2))) + else: + phase_subphases[3] = max(phase_subphases[3], int(m.group(2))-1) + else: + phase_subphases[3] = 1 + else: # CHIA: "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" m = re.match(r'^Starting phase (\d).*', line) From 6b11370a020d59917a04eaf51c93f1b7e02ab546 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 1 Jul 2021 13:33:57 -0400 Subject: [PATCH 102/109] 1 -> 2 --- src/plotman/reporting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 2eccb9ee..8641d63b 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -113,7 +113,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona ] except (psutil.NoSuchProcess, psutil.AccessDenied): # In case the job has disappeared - row = [j.plot_id[:8]] + (['--'] * (len(headings) - 1)) + row = [j.plot_id[:8]] + (['--'] * (len(headings) - 2)) if height: row.insert(0, '%3d' % i) From dd9b406d21e98e3a10b7d68451004b04df93f57c Mon Sep 17 00:00:00 2001 From: graemes Date: Fri, 2 Jul 2021 10:27:20 +0100 Subject: [PATCH 103/109] Apply review suggestion Co-authored-by: Kyle Altendorf --- src/plotman/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 7a23be20..e9d147e3 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -397,7 +397,7 @@ def set_phase_from_logfile(self) -> None: phase_subphases[1] = max(phase_subphases[1], (int(m.group(2))+1)) elif phase == 2: - if line.find('rewrite') > 0: + if 'rewrite' in line: phase_subphases[2] = max(phase_subphases[2], (9 - int(m.group(2)))) else: phase_subphases[2] = max(phase_subphases[2], (8 - int(m.group(2)))) From 73c3fe4231e6ff8785108d4064d7360a10bb4a4d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 2 Jul 2021 09:46:22 -0400 Subject: [PATCH 104/109] add -c/--contract argument parsing for madMAx processes --- src/plotman/madmax.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py index c6ba0626..a4a2e026 100644 --- a/src/plotman/madmax.py +++ b/src/plotman/madmax.py @@ -67,6 +67,8 @@ def latest_command(self) -> CommandProtocol: type=str, default=None) @click.option("-f", "--farmerkey", help="Farmer Public Key (48 bytes)", type=str, default=None) +@click.option("-c", "--contract", help="Pool Contract Address (64 chars)", + type=str, default=None) @click.option("-G", "--tmptoggle", help="Alternate tmpdir/tmpdir2", type=str, default=None) def _cli_c8121b9() -> None: From c989b1edcd1c196a104343582d2c7ae3434156c5 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 2 Jul 2021 21:41:55 -0400 Subject: [PATCH 105/109] set v0.5 --- CHANGELOG.md | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ad220e..ff0fc234 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [0.5] - 2021-07-02 ### Fixed - `plotman kill` doesn't leave any temporary files behind anymore. ([#801](https://github.com/ericaltendorf/plotman/pull/801)) diff --git a/VERSION b/VERSION index 0fac0690..2eb3c4fe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.1+dev +0.5 From 9b97471619d3102550aa08a5170425871c765f17 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 3 Jul 2021 20:04:06 -0400 Subject: [PATCH 106/109] Update pull_request_template.md --- .github/pull_request_template.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9cd4ac83..6e8b593a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1 +1,9 @@ - + From be8dd1be0478b8ba6f27818ac60b01135f69fd58 Mon Sep 17 00:00:00 2001 From: Matus Kral Date: Wed, 7 Jul 2021 00:01:42 +0200 Subject: [PATCH 107/109] - fix Job.init_from_logfile for MadMax, currently taking 3s / process --- src/plotman/job.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index e9d147e3..c016c1a8 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -337,15 +337,17 @@ def init_from_logfile(self) -> None: if m: # MADMAX self.plot_id = m.group(7) self.plotter = 'madmax' + self.start_time = pendulum.from_timestamp(os.path.getctime(self.logfile)) found_id = True + found_log = True + break + m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line) if m: # CHIA # Mon Nov 2 08:39:53 2020 self.start_time = parse_chia_plot_time(m.group(1)) found_log = True break # Stop reading lines in file - else: # MADMAX - self.start_time = pendulum.from_timestamp(os.path.getctime(self.logfile)) if found_id and found_log: break # Stop trying From 28bac66c676cd79f5e090b6480899174b7168c73 Mon Sep 17 00:00:00 2001 From: Matus Kral Date: Wed, 7 Jul 2021 00:24:50 +0200 Subject: [PATCH 108/109] - fix Job.is_plotting_cmdline for MadMax - only basename of the process name should be compared (it can contain full path to binary, or './' etc) --- src/plotman/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index e9d147e3..d04b9466 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -39,7 +39,7 @@ def is_plotting_cmdline(cmdline: typing.List[str]) -> bool: and 'plots' == cmdline[1] and 'create' == cmdline[2] ) - elif cmdline and 'chia_plot' == cmdline[0].lower(): # Madmax plotter + elif cmdline and 'chia_plot' == os.path.basename(cmdline[0].lower()): # Madmax plotter return True return False From 0f06c1126f050ffcc285cb29bf363cfab59f07fd Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 7 Jul 2021 11:49:55 -0400 Subject: [PATCH 109/109] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff0fc234..9a4d7eba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.5] - 2021-07-02 +## [0.5] - 2021-07-07 ### Fixed - `plotman kill` doesn't leave any temporary files behind anymore. ([#801](https://github.com/ericaltendorf/plotman/pull/801))