diff --git a/README.md b/README.md index 9fb73c03..094be984 100644 --- a/README.md +++ b/README.md @@ -2,5 +2,5 @@ check50 is a testing tool for checking student code. As a student you can use check50 to check your CS50 problem sets or any other Problem sets for which check50 checks exist. check50 allows teachers to automatically grade code on correctness and to provide automatic feedback while students are coding. -You can find documentation and instructions for writing your own checks at https://cs50.readthedocs.io/check50/. +You can find documentation and instructions for writing your own checks at https://cs50.readthedocs.io/projects/check50/. diff --git a/check50/__init__.py b/check50/__init__.py index 2249d6d5..558597b0 100644 --- a/check50/__init__.py +++ b/check50/__init__.py @@ -41,12 +41,13 @@ def _setup_translation(): run, log, _log, hidden, - Failure, Mismatch + Failure, Mismatch, Missing ) +from . import regex from .runner import check from pexpect import EOF -__all__ = ["import_checks", "data", "exists", "hash", "include", - "run", "log", "Failure", "Mismatch", "check", "EOF"] +__all__ = ["import_checks", "data", "exists", "hash", "include", "regex", + "run", "log", "Failure", "Mismatch", "Missing", "check", "EOF"] diff --git a/check50/__main__.py b/check50/__main__.py index e0cac2bc..2c888927 100644 --- a/check50/__main__.py +++ b/check50/__main__.py @@ -1,10 +1,10 @@ import argparse import contextlib +import enum import gettext import importlib import inspect import itertools -import json import logging import os import site @@ -14,7 +14,6 @@ import subprocess import sys import tempfile -import traceback import time import attr @@ -22,18 +21,36 @@ import requests import termcolor -from . import internal, renderer, __version__ +from . import _exceptions, internal, renderer, __version__ from .runner import CheckRunner +LOGGER = logging.getLogger("check50") + lib50.set_local_path(os.environ.get("CHECK50_PATH", "~/.local/share/check50")) -SLUG = None +class LogLevel(enum.IntEnum): + DEBUG = logging.DEBUG + INFO = logging.INFO + WARNING = logging.WARNING + ERROR = logging.ERROR + + +class ColoredFormatter(logging.Formatter): + COLORS = { + "ERROR": "red", + "WARNING": "yellow", + "DEBUG": "cyan", + "INFO": "magenta", + } + + def __init__(self, fmt, use_color=True): + super().__init__(fmt=fmt) + self.use_color = use_color -class RemoteCheckError(internal.Error): - def __init__(self, remote_json): - super().__init__("check50 ran into an error while running checks! Please contact sysadmins@cs50.harvard.edu!") - self.payload = {"remote_json": remote_json} + def format(self, record): + msg = super().format(record) + return msg if not self.use_color else termcolor.colored(msg, getattr(record, "color", self.COLORS.get(record.levelname))) @contextlib.contextmanager @@ -42,76 +59,15 @@ def nullcontext(entry_result=None): yield entry_result -def excepthook(cls, exc, tb): - # All channels to output to - outputs = excepthook.outputs - - for output in excepthook.outputs: - outputs.remove(output) - if output == "json": - ctxmanager = open(excepthook.output_file, "w") if excepthook.output_file else nullcontext(sys.stdout) - with ctxmanager as output_file: - json.dump({ - "slug": SLUG, - "error": { - "type": cls.__name__, - "value": str(exc), - "traceback": traceback.format_tb(exc.__traceback__), - "data" : exc.payload if hasattr(exc, "payload") else {} - }, - "version": __version__ - }, output_file, indent=4) - output_file.write("\n") - - elif output == "ansi" or output == "html": - if (issubclass(cls, internal.Error) or issubclass(cls, lib50.Error)) and exc.args: - termcolor.cprint(str(exc), "red", file=sys.stderr) - elif issubclass(cls, FileNotFoundError): - termcolor.cprint(_("{} not found").format(exc.filename), "red", file=sys.stderr) - elif issubclass(cls, KeyboardInterrupt): - termcolor.cprint(f"check cancelled", "red") - elif not issubclass(cls, Exception): - # Class is some other BaseException, better just let it go - return - else: - termcolor.cprint(_("Sorry, something's wrong! Let sysadmins@cs50.harvard.edu know!"), "red", file=sys.stderr) - - if excepthook.verbose: - traceback.print_exception(cls, exc, tb) - if hasattr(exc, "payload"): - print("Exception payload:", json.dumps(exc.payload), sep="\n") - - sys.exit(1) - - -def yes_no_prompt(prompt): - """ - Raise a prompt, returns True if yes is entered, False if no is entered. - Will reraise prompt in case of any other reply. - """ - yes = {"yes", "ye", "y", ""} - no = {"no", "n"} - - reply = None - while reply not in yes and reply not in no: - reply = input(f"{prompt} [Y/n] ").lower() - - return reply in yes - -# Assume we should print tracebacks until we get command line arguments -excepthook.verbose = True -excepthook.output = "ansi" -excepthook.output_file = None -sys.excepthook = excepthook +_exceptions.ExceptHook.initialize() -def install_dependencies(dependencies, verbose=False): +def install_dependencies(dependencies): """Install all packages in dependency list via pip.""" if not dependencies: return - stdout = stderr = None if verbose else subprocess.DEVNULL with tempfile.TemporaryDirectory() as req_dir: req_file = Path(req_dir) / "requirements.txt" @@ -125,9 +81,11 @@ def install_dependencies(dependencies, verbose=False): pip.append("--user") try: - subprocess.check_call(pip, stdout=stdout, stderr=stderr) + output = subprocess.check_output(pip, stderr=subprocess.STDOUT) except subprocess.CalledProcessError: - raise internal.Error(_("failed to install dependencies")) + raise _exceptions.Error(_("failed to install dependencies")) + + LOGGER.info(output) # Reload sys.path, to find recently installed packages importlib.reload(site) @@ -149,7 +107,7 @@ def install_translations(config): def compile_checks(checks, prompt=False): # Prompt to replace __init__.py (compile destination) if prompt and os.path.exists(internal.check_dir / "__init__.py"): - if not yes_no_prompt("check50 will compile the YAML checks to __init__.py, are you sure you want to overwrite its contents?"): + if not internal._yes_no_prompt("check50 will compile the YAML checks to __init__.py, are you sure you want to overwrite its contents?"): raise Error("Aborting: could not overwrite to __init__.py") # Compile simple checks @@ -165,20 +123,19 @@ def setup_logging(level): level 'info' logs all git commands run to stderr level 'debug' logs all git commands and their output to stderr """ - # No verbosity level set, don't log anything - if not level: - return - lib50_logger = logging.getLogger("lib50") + for logger in (logging.getLogger("lib50"), LOGGER): + # Set verbosity level on the lib50 logger + logger.setLevel(level.upper()) - # Set verbosity level on the lib50 logger - lib50_logger.setLevel(getattr(logging, level.upper())) + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(ColoredFormatter("(%(levelname)s) %(message)s", use_color=sys.stderr.isatty())) - # Direct all logs to sys.stderr - lib50_logger.addHandler(logging.StreamHandler(sys.stderr)) + # Direct all logs to sys.stderr + logger.addHandler(handler) - # Don't animate the progressbar - lib50.ProgressBar.DISABLED = True + # Don't animate the progressbar if LogLevel is either info or debug + lib50.ProgressBar.DISABLED = logger.level < LogLevel.WARNING def await_results(commit_hash, slug, pings=45, sleep=2): @@ -193,22 +150,22 @@ def await_results(commit_hash, slug, pings=45, sleep=2): results = res.json() if res.status_code not in [404, 200]: - raise RemoteCheckError(results) + raise _exceptions.RemoteCheckError(results) if res.status_code == 200 and results["received_at"] is not None: break time.sleep(sleep) else: # Terminate if no response - raise internal.Error( + raise _exceptions.Error( _("check50 is taking longer than normal!\n" "See https://submit.cs50.io/check50/{} for more detail").format(commit_hash)) if not results["check50"]: - raise RemoteCheckError(results) + raise _exceptions.RemoteCheckError(results) if "error" in results["check50"]: - raise RemoteCheckError(results["check50"]) + raise _exceptions.RemoteCheckError(results["check50"]) # TODO: Should probably check payload["version"] here to make sure major version is same as __version__ # (otherwise we may not be able to parse results) @@ -229,7 +186,7 @@ def __call__(self, parser, namespace, values, option_string=None): try: lib50.logout() except lib50.Error: - raise internal.Error(_("failed to logout")) + raise _exceptions.Error(_("failed to logout")) else: termcolor.cprint(_("logged out successfully"), "green") parser.exit() @@ -250,26 +207,85 @@ def raise_invalid_slug(slug, offline=False): msg += _("\nIf you are confident the slug is correct and you have an internet connection," \ " try running without --offline.") - raise internal.Error(msg) + raise _exceptions.Error(msg) + + +def process_args(args): + """Validate arguments and apply defaults that are dependent on other arguments""" + + # dev implies offline, verbose, and log level "INFO" if not overwritten + if args.dev: + args.offline = True + if "ansi" in args.output: + args.ansi_log = True + + if not args.log_level: + args.log_level = "info" + + # offline implies local + if args.offline: + args.no_install_dependencies = True + args.no_download_checks = True + args.local = True + + if not args.log_level: + args.log_level = "warning" + + # Setup logging for lib50 + setup_logging(args.log_level) + + # Warning in case of running remotely with no_download_checks or no_install_dependencies set + if not args.local: + useless_args = [] + if args.no_download_checks: + useless_args.append("--no-downloads-checks") + if args.no_install_dependencies: + useless_args.append("--no-install-dependencies") + + if useless_args: + LOGGER.warning(_("You should always use --local when using: {}").format(", ".join(useless_args))) + + # Filter out any duplicates from args.output + seen_output = [] + for output in args.output: + if output in seen_output: + LOGGER.warning(_("Duplicate output format specified: {}").format(output)) + else: + seen_output.append(output) + + args.output = seen_output + + if args.ansi_log and "ansi" not in seen_output: + LOGGER.warning(_("--ansi-log has no effect when ansi is not among the output formats")) + + +class LoggerWriter: + def __init__(self, logger, level): + self.logger = logger + self.level = level + + def write(self, message): + if message != "\n": + self.logger.log(self.level, message) + + def flush(self): + pass def main(): - parser = argparse.ArgumentParser(prog="check50") + parser = argparse.ArgumentParser(prog="check50", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("slug", help=_("prescribed identifier of work to check")) parser.add_argument("-d", "--dev", action="store_true", - help=_("run check50 in development mode (implies --offline and --verbose info).\n" - "causes SLUG to be interpreted as a literal path to a checks package")) + help=_("run check50 in development mode (implies --offline, and --log-level info).\n" + "causes slug to be interpreted as a literal path to a checks package.")) parser.add_argument("--offline", action="store_true", help=_("run checks completely offline (implies --local, --no-download-checks and --no-install-dependencies)")) parser.add_argument("-l", "--local", action="store_true", help=_("run checks locally instead of uploading to cs50")) - parser.add_argument("--log", - action="store_true", - help=_("display more detailed information about check results")) parser.add_argument("-o", "--output", action="store", nargs="+", @@ -284,16 +300,16 @@ def main(): action="store", metavar="FILE", help=_("file to write output to")) - parser.add_argument("-v", "--verbose", + parser.add_argument("--log-level", action="store", - nargs="?", - default="", - const="info", - choices=[attr for attr in dir(logging) if attr.isupper() and isinstance(getattr(logging, attr), int)], - type=str.upper, - help=_("sets the verbosity level." - ' "INFO" displays the full tracebacks of errors and shows all commands run.' - ' "DEBUG" adds the output of all command run.')) + choices=[level.name.lower() for level in LogLevel], + type=str.lower, + help=_('warning: displays usage warnings.' + '\ninfo: adds all commands run, any locally installed dependencies and print messages.' + '\ndebug: adds the output of all commands run.')) + parser.add_argument("--ansi-log", + action="store_true", + help=_("display log in ansi output mode")) parser.add_argument("--no-download-checks", action="store_true", help=_("do not download checks, but use previously downloaded checks instead (only works with --local)")) @@ -307,66 +323,36 @@ def main(): args = parser.parse_args() - global SLUG - SLUG = args.slug - - # dev implies offline and verbose "info" if not overwritten - if args.dev: - args.offline = True - if not args.verbose: - args.verbose = "info" - - # offline implies local - if args.offline: - args.no_install_dependencies = True - args.no_download_checks = True - args.local = True - - # Setup logging for lib50 depending on verbosity level - setup_logging(args.verbose) - - # Warning in case of running remotely with no_download_checks or no_install_dependencies set - if not args.local: - useless_args = [] - if args.no_download_checks: - useless_args.append("--no-downloads-checks") - if args.no_install_dependencies: - useless_args.append("--no-install-dependencies") - - if useless_args: - termcolor.cprint(_("Warning: you should always use --local when using: {}".format(", ".join(useless_args))), - "yellow", attrs=["bold"]) + internal.slug = args.slug - # Filter out any duplicates from args.output - seen_output = set() - args.output = [output for output in args.output if not (output in seen_output or seen_output.add(output))] + # Validate arguments and apply defaults + process_args(args) # Set excepthook - excepthook.verbose = bool(args.verbose) - excepthook.outputs = args.output - excepthook.output_file = args.output_file + _exceptions.ExceptHook.initialize(args.output, args.output_file) # If remote, push files to GitHub and await results if not args.local: - commit_hash = lib50.push("check50", SLUG, internal.CONFIG_LOADER, data={"check50": True})[1] + commit_hash = lib50.push("check50", internal.slug, internal.CONFIG_LOADER, data={"check50": True})[1] with lib50.ProgressBar("Waiting for results") if "ansi" in args.output else nullcontext(): - tag_hash, results = await_results(commit_hash, SLUG) + tag_hash, results = await_results(commit_hash, internal.slug) + # Otherwise run checks locally else: with lib50.ProgressBar("Checking") if "ansi" in args.output else nullcontext(): # If developing, assume slug is a path to check_dir if args.dev: - internal.check_dir = Path(SLUG).expanduser().resolve() + internal.check_dir = Path(internal.slug).expanduser().resolve() if not internal.check_dir.is_dir(): - raise internal.Error(_("{} is not a directory").format(internal.check_dir)) + raise _exceptions.Error(_("{} is not a directory").format(internal.check_dir)) # Otherwise have lib50 create a local copy of slug else: try: - internal.check_dir = lib50.local(SLUG, offline=args.no_download_checks) + internal.check_dir = lib50.local(internal.slug, offline=args.no_download_checks) except lib50.ConnectionError: - raise internal.Error(_("check50 could not retrieve checks from GitHub. Try running check50 again with --offline.").format(SLUG)) + raise _exceptions.Error(_("check50 could not retrieve checks from GitHub. Try running check50 again with --offline.").format(internal.slug)) except lib50.InvalidSlugError: - raise_invalid_slug(SLUG, offline=args.no_download_checks) + raise_invalid_slug(internal.slug, offline=args.no_download_checks) # Load config config = internal.load_config(internal.check_dir) @@ -378,33 +364,26 @@ def main(): install_translations(config["translations"]) if not args.no_install_dependencies: - install_dependencies(config["dependencies"], verbose=args.verbose) + install_dependencies(config["dependencies"]) checks_file = (internal.check_dir / config["checks"]).resolve() # Have lib50 decide which files to include - included = lib50.files(config.get("files"))[0] + included_files = lib50.files(config.get("files"))[0] - with open(os.devnull, "w") if args.verbose else nullcontext() as devnull: - # Redirect stdout to devnull if some verbosity level is set - if args.verbose: - stdout = stderr = devnull - else: - stdout = sys.stdout - stderr = sys.stderr + # Create a working_area (temp dir) named - with all included student files + with CheckRunner(checks_file, included_files) as check_runner, \ + contextlib.redirect_stdout(LoggerWriter(LOGGER, logging.INFO)), \ + contextlib.redirect_stderr(LoggerWriter(LOGGER, logging.INFO)): - # Create a working_area (temp dir) named - with all included student files - with lib50.working_area(included, name='-') as working_area, \ - contextlib.redirect_stdout(stdout), \ - contextlib.redirect_stderr(stderr): - - check_results = CheckRunner(checks_file).run(included, working_area, args.target) - results = { - "slug": SLUG, - "results": [attr.asdict(result) for result in check_results], - "version": __version__ - } + check_results = check_runner.run(args.target) + results = { + "slug": internal.slug, + "results": [attr.asdict(result) for result in check_results], + "version": __version__ + } + LOGGER.debug(results) # Render output file_manager = open(args.output_file, "w") if args.output_file else nullcontext(sys.stdout) @@ -414,7 +393,7 @@ def main(): output_file.write(renderer.to_json(**results)) output_file.write("\n") elif output == "ansi": - output_file.write(renderer.to_ansi(**results, log=args.log)) + output_file.write(renderer.to_ansi(**results, _log=args.ansi_log)) output_file.write("\n") elif output == "html": if os.environ.get("CS50_IDE_TYPE") and args.local: diff --git a/check50/_api.py b/check50/_api.py index abd2d69e..f655b1b3 100644 --- a/check50/_api.py +++ b/check50/_api.py @@ -1,6 +1,8 @@ import hashlib import functools +import numbers import os +import re import shlex import shutil import signal @@ -10,7 +12,7 @@ import pexpect from pexpect.exceptions import EOF, TIMEOUT -from . import internal +from . import internal, regex _log = [] internal.register.before_every(_log.clear) @@ -135,6 +137,7 @@ def import_checks(path): return mod + class run: """ Run a command. @@ -163,12 +166,15 @@ def __init__(self, command, env={}): command = "bash -c {}".format(shlex.quote(command)) self.process = pexpect.spawn(command, encoding="utf-8", echo=False, env=full_env) - def stdin(self, line, prompt=True, timeout=3): + def stdin(self, line, str_line=None, prompt=True, timeout=3): """ Send line to stdin, optionally expect a prompt. :param line: line to be send to stdin :type line: str + :param str_line: what will be displayed as the delivered input, a human \ + readable form of ``line`` + :type str_line: str :param prompt: boolean indicating whether a prompt is expected, if True absorbs \ all of stdout before inserting line into stdin and raises \ :class:`check50.Failure` if stdout is empty @@ -178,10 +184,13 @@ def stdin(self, line, prompt=True, timeout=3): :raises check50.Failure: if ``prompt`` is set to True and no prompt is given """ + if str_line is None: + str_line = line + if line == EOF: log("sending EOF...") else: - log(_("sending input {}...").format(line)) + log(_("sending input {}...").format(str_line)) if prompt: try: @@ -207,15 +216,18 @@ def stdin(self, line, prompt=True, timeout=3): pass return self - def stdout(self, output=None, str_output=None, regex=True, timeout=3): + def stdout(self, output=None, str_output=None, regex=True, timeout=3, show_timeout=False): """ Retrieve all output from stdout until timeout (3 sec by default). If ``output`` is None, ``stdout`` returns all of the stdout outputted by the process, else it returns ``self``. :param output: optional output to be expected from stdout, raises \ - :class:`check50.Failure` if no match - :type output: str + :class:`check50.Failure` if no match \ + In case output is a float or int, the check50.number_regex \ + is used to match just that number". \ + In case output is a stream its contents are used via output.read(). + :type output: str, int, float, stream :param str_output: what will be displayed as expected output, a human \ readable form of ``output`` :type str_output: str @@ -223,6 +235,9 @@ def stdout(self, output=None, str_output=None, regex=True, timeout=3): :type regex: bool :param timeout: maximum number of seconds to wait for ``output`` :type timeout: int / float + :param show_timeout: flag indicating whether the timeout in seconds \ + should be displayed when a timeout occurs + :type show_timeout: bool :raises check50.Mismatch: if ``output`` is specified and nothing that the \ process outputs matches it :raises check50.Failure: if process times out or if it outputs invalid UTF-8 text. @@ -239,20 +254,26 @@ def stdout(self, output=None, str_output=None, regex=True, timeout=3): self._wait(timeout) return self.process.before.replace("\r\n", "\n").lstrip("\n") + # In case output is a stream (file-like object), read from it try: output = output.read() except AttributeError: pass - expect = self.process.expect if regex else self.process.expect_exact - if str_output is None: - str_output = output + str_output = str(output) + + # In case output is an int/float, use a regex to match exactly that int/float + if isinstance(output, numbers.Number): + regex = True + output = globals()["regex"].decimal(output) + + expect = self.process.expect if regex else self.process.expect_exact if output == EOF: log(_("checking for EOF...")) else: - output = output.replace("\n", "\r\n") + output = str(output).replace("\n", "\r\n") log(_("checking for output \"{}\"...").format(str_output)) try: @@ -263,6 +284,9 @@ def stdout(self, output=None, str_output=None, regex=True, timeout=3): result += self.process.after raise Mismatch(str_output, result.replace("\r\n", "\n")) except TIMEOUT: + if show_timeout: + raise Missing(str_output, self.process.before, + help=_("check50 waited {} seconds for the output of the program").format(timeout)) raise Missing(str_output, self.process.before) except UnicodeDecodeError: raise Failure(_("output not valid ASCII text")) @@ -401,6 +425,10 @@ class Missing(Failure): def __init__(self, missing_item, collection, help=None): super().__init__(rationale=_("Did not find {} in {}").format(_raw(missing_item), _raw(collection)), help=help) + + if missing_item == EOF: + missing_item = "EOF" + self.payload.update({"missing_item": str(missing_item), "collection": str(collection)}) diff --git a/check50/_exceptions.py b/check50/_exceptions.py new file mode 100644 index 00000000..2191e134 --- /dev/null +++ b/check50/_exceptions.py @@ -0,0 +1,83 @@ +import json +import sys +import traceback + +import lib50 +import termcolor + +from . import internal, __version__ + +class Error(Exception): + """Exception for internal check50 errors.""" + pass + + +class RemoteCheckError(Error): + """An exception for errors that happen in check50's remote operation.""" + def __init__(self, remote_json): + super().__init__("check50 ran into an error while running checks! Please contact sysadmins@cs50.harvard.edu!") + self.payload = {"remote_json": remote_json} + + +class ExceptHook: + def __init__(self, outputs=("ansi",), output_file=None): + self.outputs = outputs + self.output_file = output_file + + def __call__(self, cls, exc, tb): + # If an error happened remotely, grab its traceback and message + if issubclass(cls, RemoteCheckError) and "error" in exc.payload["remote_json"]: + formatted_traceback = exc.payload["remote_json"]["error"]["traceback"] + show_traceback = exc.payload["remote_json"]["error"]["actions"]["show_traceback"] + message = exc.payload["remote_json"]["error"]["actions"]["message"] + # Otherwise, create the traceback and message from this error + else: + formatted_traceback = traceback.format_exception(cls, exc, tb) + show_traceback = False + + if (issubclass(cls, Error) or issubclass(cls, lib50.Error)) and exc.args: + message = str(exc) + elif issubclass(cls, FileNotFoundError): + message = _("{} not found").format(exc.filename) + elif issubclass(cls, KeyboardInterrupt): + message = _("check cancelled") + elif not issubclass(cls, Exception): + # Class is some other BaseException, better just let it go + return + else: + show_traceback = True + message = _("Sorry, something is wrong! check50 ran into an error.\n" \ + "Please let CS50 know by emailing the error above to sysadmins@cs50.harvard.edu.") + + # Output exception as json + if "json" in self.outputs: + ctxmanager = open(self.output_file, "w") if self.output_file else nullcontext(sys.stdout) + with ctxmanager as output_file: + json.dump({ + "slug": internal.slug, + "error": { + "type": cls.__name__, + "value": str(exc), + "traceback": formatted_traceback, + "actions": { + "show_traceback": show_traceback, + "message": message + }, + "data" : exc.payload if hasattr(exc, "payload") else {} + }, + "version": __version__ + }, output_file, indent=4) + output_file.write("\n") + + # Output exception to stderr + if "ansi" in self.outputs or "html" in self.outputs: + if show_traceback: + for line in formatted_traceback: + termcolor.cprint(line, end="", file=sys.stderr) + termcolor.cprint(message, "red", file=sys.stderr) + + sys.exit(1) + + @classmethod + def initialize(cls, *args, **kwargs): + sys.excepthook = cls(*args, **kwargs) diff --git a/check50/c.py b/check50/c.py index 26b195a5..3fd5ca79 100644 --- a/check50/c.py +++ b/check50/c.py @@ -14,7 +14,7 @@ CFLAGS = {"std": "c11", "ggdb": True, "lm": True} -def compile(*files, exe_name=None, cc=CC, **cflags): +def compile(*files, exe_name=None, cc=CC, max_log_lines=50, **cflags): """ Compile C source files. @@ -60,9 +60,16 @@ def compile(*files, exe_name=None, cc=CC, **cflags): # Strip out ANSI codes stdout = re.sub(r"\x1B\[[0-?]*[ -/]*[@-~]", "", process.stdout()) + # Log max_log_lines lines of output in case compilation fails if process.exitcode != 0: - for line in stdout.splitlines(): + lines = stdout.splitlines() + + if len(lines) > max_log_lines: + lines = lines[:max_log_lines // 2] + lines[-(max_log_lines // 2):] + + for line in lines: log(line) + raise Failure("code failed to compile") @@ -130,4 +137,4 @@ def _check_valgrind(xml_file): # Only raise exception if we encountered errors. if reported: - raise Failure(_("valgrind tests failed; rerun with --log for more information.")) + raise Failure(_("valgrind tests failed; see log for more information.")) diff --git a/check50/flask.py b/check50/flask.py index 5944bfc6..3ee266db 100644 --- a/check50/flask.py +++ b/check50/flask.py @@ -142,7 +142,7 @@ def _send(self, method, route, data, params, **kwargs): self.response = getattr(self._client, method.lower())(route, data=data, **kwargs) except BaseException as e: # Catch all exceptions thrown by app log(_("exception raised in application: {}: {}").format(type(e).__name__, e)) - raise Failure(_("application raised an exception (rerun with --log for more details)")) + raise Failure(_("application raised an exception (see the log for more details)")) return self def _search_page(self, output, str_output, content, match_fn, **kwargs): diff --git a/check50/internal.py b/check50/internal.py index b5996c45..ae120189 100644 --- a/check50/internal.py +++ b/check50/internal.py @@ -2,33 +2,40 @@ Additional check50 internals exposed to extension writers in addition to the standard API """ -import importlib from pathlib import Path +import importlib +import json import sys +import termcolor +import traceback import lib50 -from . import _simple +from . import _simple, _exceptions #: Directory containing the check and its associated files check_dir = None -#: Temporary directory in which check is being run +#: Temporary directory in which the current check is being run run_dir = None +#: Temporary directory that is the root (parent) of all run_dir(s) +run_root_dir = None + +#: Directory check50 was run from +student_dir = None + #: Boolean that indicates if a check is currently running check_running = False +#: The user specified slug used to indentifies the set of checks +slug = None + #: ``lib50`` config loader CONFIG_LOADER = lib50.config.Loader("check50") CONFIG_LOADER.scope("files", "include", "exclude", "require") -class Error(Exception): - """Exception for internal check50 errors.""" - pass - - class Register: """ Class with which functions can be registered to run before / after checks. @@ -45,7 +52,7 @@ def after_check(self, func): :param func: callback to run after check :raises check50.internal.Error: if called when no check is being run""" if not check_running: - raise Error("cannot register callback to run after check when no check is running") + raise _exceptions.Error("cannot register callback to run after check when no check is running") self._after_checks.append(func) def after_every(self, func): @@ -54,7 +61,7 @@ def after_every(self, func): :param func: callback to be run after every check :raises check50.internal.Error: if called when a check is being run""" if check_running: - raise Error("cannot register callback to run after every check when check is running") + raise _exceptions.Error("cannot register callback to run after every check when check is running") self._after_everies.append(func) def before_every(self, func): @@ -64,7 +71,7 @@ def before_every(self, func): :raises check50.internal.Error: if called when a check is being run""" if check_running: - raise Error("cannot register callback to run before every check when check is running") + raise _exceptions.Error("cannot register callback to run before every check when check is running") self._before_everies.append(func) def __enter__(self): @@ -115,14 +122,14 @@ def load_config(check_dir): try: config_file = lib50.config.get_config_filepath(check_dir) except lib50.Error: - raise Error(_("Invalid slug for check50. Did you mean something else?")) + raise _exceptions.Error(_("Invalid slug for check50. Did you mean something else?")) # Load config with open(config_file) as f: try: config = CONFIG_LOADER.load(f.read()) except lib50.InvalidConfigError: - raise Error(_("Invalid slug for check50. Did you mean something else?")) + raise _exceptions.Error(_("Invalid slug for check50. Did you mean something else?")) # Update the config with defaults if isinstance(config, dict): @@ -155,7 +162,7 @@ def compile_checks(checks, prompt=False, out_file="__init__.py"): # Prompt to replace __init__.py (compile destination) if prompt and file_path.exists(): if not _yes_no_prompt("check50 will compile the YAML checks to __init__.py, are you sure you want to overwrite its contents?"): - raise Error("Aborting: could not overwrite to __init__.py") + raise _exceptions.Error("Aborting: could not overwrite to __init__.py") # Compile simple checks with open(check_dir / out_file, "w") as f: diff --git a/check50/py.py b/check50/py.py index 66f6ed89..167de3f8 100644 --- a/check50/py.py +++ b/check50/py.py @@ -1,7 +1,6 @@ import importlib from pathlib import Path import py_compile -import traceback from . import internal from ._api import Failure, exists, log @@ -65,5 +64,4 @@ def compile(file): for line in e.msg.splitlines(): log(line) - raise Failure(_("{} raised while compiling {} (rerun with --log for more details)").format(e.exc_type_name, file)) - + raise Failure(_("{} raised while compiling {} (see the log for more details)").format(e.exc_type_name, file)) diff --git a/check50/regex.py b/check50/regex.py new file mode 100644 index 00000000..215c65a8 --- /dev/null +++ b/check50/regex.py @@ -0,0 +1,33 @@ +import re + + +def decimal(number): + """ + Create a regular expression to match the number exactly: + + In case of a positive number:: + + (?= 0 else "" + return fr"{negative_lookbehind}{re.escape(str(number))}(?!(\.?\d))" diff --git a/check50/renderer/_renderers.py b/check50/renderer/_renderers.py index 53512dd2..bbbb43ef 100644 --- a/check50/renderer/_renderers.py +++ b/check50/renderer/_renderers.py @@ -23,7 +23,7 @@ def to_json(slug, results, version): return json.dumps({"slug": slug, "results": results, "version": version}, indent=4) -def to_ansi(slug, results, version, log=False): +def to_ansi(slug, results, version, _log=False): lines = [termcolor.colored(_("Results for {} generated by check50 v{}").format(slug, version), "white", attrs=["bold"])] for result in results: if result["passed"]: @@ -41,7 +41,7 @@ def to_ansi(slug, results, version, log=False): if result["cause"].get("help") is not None: lines.append(termcolor.colored(f" {result['cause']['help']}", "red")) - if log: + if _log: lines += (f" {line}" for line in result["log"]) return "\n".join(lines) diff --git a/check50/renderer/templates/results.html b/check50/renderer/templates/results.html index 683ef88f..6a51efa8 100644 --- a/check50/renderer/templates/results.html +++ b/check50/renderer/templates/results.html @@ -43,7 +43,7 @@

:| {{ check.description }}

{% if check.log %} Log
-
+
{% for item in check.log %} {{ item }}
diff --git a/check50/runner.py b/check50/runner.py index d0d7f44d..dd4a1fc9 100644 --- a/check50/runner.py +++ b/check50/runner.py @@ -10,12 +10,14 @@ from pathlib import Path import shutil import signal +import sys import tempfile import traceback import attr +import lib50 -from . import internal +from . import internal, __version__ from ._api import log, Failure, _copy, _log, _data _check_names = [] @@ -80,13 +82,15 @@ def _handle_timeout(*args): signal.signal(signal.SIGALRM, signal.SIG_DFL) -def check(dependency=None, timeout=60): +def check(dependency=None, timeout=60, max_log_lines=100): """Mark function as a check. :param dependency: the check that this check depends on :type dependency: function :param timeout: maximum number of seconds the check can run :type timeout: int + :param max_log_lines: maximum number of lines that can appear in the log + :type max_log_lines: int When a check depends on another, the former will only run if the latter passes. Additionally, the dependent check will inherit the filesystem of its dependency. @@ -123,7 +127,7 @@ def decorator(check): check._check_dependency = dependency @functools.wraps(check) - def wrapper(checks_root, dependency_state): + def wrapper(run_root_dir, dependency_state): # Result template result = CheckResult.from_check(check) # Any shared (returned) state @@ -131,8 +135,8 @@ def wrapper(checks_root, dependency_state): try: # Setup check environment, copying disk state from dependency - internal.run_dir = checks_root / check.__name__ - src_dir = checks_root / (dependency.__name__ if dependency else "-") + internal.run_dir = run_root_dir / check.__name__ + src_dir = run_root_dir / (dependency.__name__ if dependency else "-") shutil.copytree(src_dir, internal.run_dir) os.chdir(internal.run_dir) @@ -155,7 +159,7 @@ def wrapper(checks_root, dependency_state): else: result.passed = True finally: - result.log = _log + result.log = _log if len(_log) <= max_log_lines else ["..."] + _log[-max_log_lines:] result.data = _data return result, state return wrapper @@ -163,33 +167,11 @@ def wrapper(checks_root, dependency_state): class CheckRunner: - def __init__(self, checks_path): - # TODO: Naming the module "checks" is arbitray. Better name? - self.checks_spec = importlib.util.spec_from_file_location("checks", checks_path) + def __init__(self, checks_path, included_files): + self.checks_path = checks_path + self.included_files = included_files - # Clear check_names, import module, then save check_names. Not thread safe. - # Ideally, there'd be a better way to extract declaration order than @check mutating global state, - # but there are a lot of subtleties with using `inspect` or similar here - _check_names.clear() - check_module = importlib.util.module_from_spec(self.checks_spec) - self.checks_spec.loader.exec_module(check_module) - self.check_names = _check_names.copy() - _check_names.clear() - - # Grab all checks from the module - checks = inspect.getmembers(check_module, lambda f: hasattr(f, "_check_dependency")) - - # Map each check to tuples containing the names of the checks that depend on it - self.dependency_graph = collections.defaultdict(set) - for name, check in checks: - dependency = None if check._check_dependency is None else check._check_dependency.__name__ - self.dependency_graph[dependency].add(name) - - # Map each check name to its description - self.check_descriptions = {name: check.__doc__ for name, check in checks} - - - def run(self, files, working_area, targets=None): + def run(self, targets=None): """ Run checks concurrently. Returns a list of CheckResults ordered by declaration order of the checks in the imported module @@ -200,7 +182,6 @@ def run(self, files, working_area, targets=None): # Ensure that dictionary is ordered by check declaration order (via self.check_names) # NOTE: Requires CPython 3.6. If we need to support older versions of Python, replace with OrderedDict. results = {name: None for name in self.check_names} - checks_root = working_area.parent try: max_workers = int(os.environ.get("CHECK50_WORKERS")) @@ -209,7 +190,7 @@ def run(self, files, working_area, targets=None): with futures.ProcessPoolExecutor(max_workers=max_workers) as executor: # Start all checks that have no dependencies - not_done = set(executor.submit(run_check(name, self.checks_spec, checks_root)) + not_done = set(executor.submit(run_check(name, self.checks_spec)) for name in graph[None]) not_passed = [] @@ -223,7 +204,7 @@ def run(self, files, working_area, targets=None): # Dispatch dependent checks for child_name in graph[result.name]: not_done.add(executor.submit( - run_check(child_name, self.checks_spec, checks_root, state))) + run_check(child_name, self.checks_spec, state))) else: not_passed.append(result.name) @@ -288,23 +269,96 @@ def _skip_children(self, check_name, results): self._skip_children(name, results) + def __enter__(self): + # Remember the student's directory + internal.student_dir = Path.cwd() + + # Set up a temp dir for the checks + self._working_area_manager = lib50.working_area(self.included_files, name='-') + internal.run_root_dir = self._working_area_manager.__enter__().parent + + # Change current working dir to the temp dir + self._cd_manager = lib50.cd(internal.run_root_dir) + self._cd_manager.__enter__() + + # TODO: Naming the module "checks" is arbitray. Better name? + self.checks_spec = importlib.util.spec_from_file_location("checks", self.checks_path) + + # Clear check_names, import module, then save check_names. Not thread safe. + # Ideally, there'd be a better way to extract declaration order than @check mutating global state, + # but there are a lot of subtleties with using `inspect` or similar here + _check_names.clear() + check_module = importlib.util.module_from_spec(self.checks_spec) + self.checks_spec.loader.exec_module(check_module) + self.check_names = _check_names.copy() + _check_names.clear() + + # Grab all checks from the module + checks = inspect.getmembers(check_module, lambda f: hasattr(f, "_check_dependency")) + + # Map each check to tuples containing the names of the checks that depend on it + self.dependency_graph = collections.defaultdict(set) + for name, check in checks: + dependency = None if check._check_dependency is None else check._check_dependency.__name__ + self.dependency_graph[dependency].add(name) + + # Map each check name to its description + self.check_descriptions = {name: check.__doc__ for name, check in checks} + + return self + + + def __exit__(self, type, value, tb): + # Destroy the temporary directory for the checks + self._working_area_manager.__exit__(type, value, tb) + + # cd back to the directory check50 was called from + self._cd_manager.__exit__(type, value, tb) + + class run_check: """ - Hack to get around the fact that `pickle` can't serialize closures. + Check job that runs in a separate process. + This is only a class to get around the fact that `pickle` can't serialize closures. This class is essentially a function that reimports the check module and runs the check. """ - def __init__(self, check_name, spec, checks_root, state=None): + # All attributes shared between check50's main process and each checks' process + # Required for "spawn": https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods + CROSS_PROCESS_ATTRIBUTES = ( + "internal.check_dir", + "internal.slug", + "internal.student_dir", + "internal.run_root_dir", + "sys.excepthook", + "__version__" + ) + + def __init__(self, check_name, spec, state=None): self.check_name = check_name self.spec = spec - self.checks_root = checks_root self.state = state + self.attribute_values = tuple(eval(name) for name in self.CROSS_PROCESS_ATTRIBUTES) + + @staticmethod + def _set_attribute(name, value): + """Get an attribute from a name in global scope and set its value.""" + parts = name.split(".") + + obj = sys.modules[__name__] + for part in parts[:-1]: + obj = getattr(obj, part) + + setattr(obj, parts[-1], value) def __call__(self): + for name, val in zip(self.CROSS_PROCESS_ATTRIBUTES, self.attribute_values): + self._set_attribute(name, val) + mod = importlib.util.module_from_spec(self.spec) self.spec.loader.exec_module(mod) internal.check_running = True try: - return getattr(mod, self.check_name)(self.checks_root, self.state) + return getattr(mod, self.check_name)(internal.run_root_dir, self.state) finally: internal.check_running = False diff --git a/docs/source/api.rst b/docs/source/api.rst index 6a778ba8..dc76c331 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -5,18 +5,21 @@ API docs .. _check50: + check50 ******* .. automodule:: check50 :members: + check50.c ********** .. automodule:: check50.c :members: + check50.flask **************** @@ -30,6 +33,12 @@ check50.py :members: +check50.regex +************** +.. automodule:: check50.regex + :members: + + check50.internal ***************** diff --git a/docs/source/check50_user.rst b/docs/source/check50_user.rst index 69e6b908..7ed0f6c4 100644 --- a/docs/source/check50_user.rst +++ b/docs/source/check50_user.rst @@ -26,19 +26,19 @@ Operation modes Check50 can run in four mutually exclusive modes of operation. ********************** -local +online ********************** -By default check50 runs locally. That means the checks run locally on the machine you run check50 on. The checks however are fetched remotely from GitHub. +By default check50 runs the checks remotely and then waits for the results to come back. ********************** -offline +local ********************** -Running with :code:`--offline` runs the checks locally and has check50 look for checks locally. check50 will not try to fetch checks remotely in this mode. +To run checks locally, pass the :code:`--local` flag. The checks are still fetched remotely from GitHub. ********************** -online +offline ********************** -Running with :code:`--online` runs the checks remotely and then waits for the results to come back. +Running with :code:`--offline` runs the checks locally and has check50 look for checks locally. check50 will not try to fetch checks remotely in this mode. ********************** dev @@ -53,12 +53,12 @@ By default check50 will try to keep its output concise in its :code:`ansi` outpu ********************** verbose ********************** -Running with :code:`--verbose` lets check50 output both the log and any tracebacks in the :code:`ansi` output mode. +Running with :code:`--verbose` lets check50 output any tracebacks in the :code:`ansi` output mode. ********************** -log +log-level ********************** -Running check50 with :code:`--log` will have check50 print out its logs. +Running check50 with :code:`--log-level INFO` will display any git commands run. :code:`--log-level DEBUG` adds all output of any git commands run. Targeting checks @@ -68,7 +68,7 @@ Check50 lets you target specific checks by name with the :code:`--target` flags. ********************** target ********************** -With :code:`--target` you can target checks from a larger body of checks by name. check50 will only run and show these checks and their dependencies. +With :code:`--target` you can target checks from a larger body of checks by name. check50 will only run and show these checks and their dependencies. Output modes diff --git a/setup.py b/setup.py index 6feb63a9..a9fe0e31 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ message_extractors = { 'check50': [('**.py', 'python', None),], }, - install_requires=["attrs>=18", "bs4>=0", "pexpect>=4.6", "lib50>=2,<4", "pyyaml>=3.10", "requests>=2.19", "termcolor>=1.1", "jinja2>=2.10"], + install_requires=["attrs>=18", "beautifulsoup4>=0", "pexpect>=4.6", "lib50>=2,<4", "pyyaml>=3.10", "requests>=2.19", "termcolor>=1.1", "jinja2>=2.10"], extras_require = { "develop": ["sphinx", "sphinx-autobuild", "sphinx_rtd_theme"] }, @@ -29,6 +29,6 @@ "console_scripts": ["check50=check50.__main__:main"] }, url="https://github.com/cs50/check50", - version="3.1.2", + version="3.2.0", include_package_data=True ) diff --git a/tests/api_tests.py b/tests/api_tests.py index 4a3ca803..bd45f140 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -163,6 +163,74 @@ def test_out_no_regex(self): self.process.stdout(".o.", regex=False) self.assertFalse(self.process.process.isalive()) + def test_int(self): + self.write("print(123)") + self.runpy() + with self.assertRaises(check50.Failure): + self.process.stdout(1) + + self.write("print(21)") + self.runpy() + with self.assertRaises(check50.Failure): + self.process.stdout(1) + + self.write("print(1.0)") + self.runpy() + with self.assertRaises(check50.Failure): + self.process.stdout(1) + + self.write("print('a1b')") + self.runpy() + self.process.stdout(1) + + self.write("print(1)") + self.runpy() + self.process.stdout(1) + + def test_float(self): + self.write("print(1.01)") + self.runpy() + with self.assertRaises(check50.Failure): + self.process.stdout(1.0) + + self.write("print(21.0)") + self.runpy() + with self.assertRaises(check50.Failure): + self.process.stdout(1.0) + + self.write("print(1)") + self.runpy() + with self.assertRaises(check50.Failure): + self.process.stdout(1.0) + + self.write("print('a1.0b')") + self.runpy() + self.process.stdout(1.0) + + self.write("print(1.0)") + self.runpy() + self.process.stdout(1.0) + + def test_negative_number(self): + self.write("print(1)") + self.runpy() + with self.assertRaises(check50.Failure): + self.process.stdout(-1) + + self.write("print(-1)") + self.runpy() + with self.assertRaises(check50.Failure): + self.process.stdout(1) + + self.write("print('2-1')") + self.runpy() + self.process.stdout(-1) + + self.write("print(-1)") + self.runpy() + self.process.stdout(-1) + + class TestProcessStdoutFile(Base): def setUp(self): super().setUp() diff --git a/tests/c_tests.py b/tests/c_tests.py index 3cb32471..6a3e02cd 100644 --- a/tests/c_tests.py +++ b/tests/c_tests.py @@ -28,6 +28,12 @@ def tearDown(self): self.working_directory.cleanup() class TestCompile(Base): + def test_compile_incorrect(self): + open("blank.c", "w").close() + + with self.assertRaises(check50.Failure): + check50.c.compile("blank.c") + def test_compile_hello_world(self): with open("hello.c", "w") as f: src = '#include \n'\ diff --git a/tests/check50_tests.py b/tests/check50_tests.py index c8a8b2e2..350495fe 100644 --- a/tests/check50_tests.py +++ b/tests/check50_tests.py @@ -58,7 +58,7 @@ def test_no_file(self): process.expect_exact("can't check until a frown turns upside down") process.close(force=True) - def test_with_file(self): + def test_with_correct_file(self): open("foo.py", "w").close() process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/exit_py") process.expect_exact(":)") @@ -67,6 +67,18 @@ def test_with_file(self): process.expect_exact("foo.py exits properly") process.close(force=True) + def test_with_incorrect_file(self): + with open("foo.py", "w") as f: + f.write("from sys import exit\nexit(1)") + + process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/exit_py") + process.expect_exact(":)") + process.expect_exact("foo.py exists") + process.expect_exact(":(") + process.expect_exact("foo.py exits properly") + process.expect_exact("expected exit code 0, not 1") + process.close(force=True) + class TestStdoutPy(Base): def test_no_file(self): @@ -102,6 +114,17 @@ def test_with_correct_file(self): process.expect_exact("prints hello") process.close(force=True) +class TestStdoutTimeout(Base): + def test_stdout_timeout(self): + with open("foo.py", "w") as f: + f.write("while True: pass") + + process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdout_py") + process.expect_exact(":)") + process.expect_exact("foo.py exists") + process.expect_exact(":(") + process.expect_exact("check50 waited 1 seconds for the output of the program") + process.close(force=True) class TestStdinPy(Base): def test_no_file(self): @@ -230,6 +253,36 @@ def test_with_correct_file(self): process.expect_exact("prints hello name (chaining) (order)") process.close(force=True) +class TestStdinHumanReadable(Base): + def test_without_human_readable_string(self): + with open("foo.py", "w") as f: + f.write('name = input()\nprint("hello", name)') + + process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_py") + process.expect_exact(":)") + process.expect_exact("foo.py exists") + process.expect_exact("checking that foo.py exists...") + process.expect_exact(":)") + process.expect_exact("prints hello name") + process.expect_exact("running python3 foo.py...") + process.expect_exact("sending input bar...") + process.expect_exact("checking for output \"hello bar\"...") + process.close(force=True) + + def test_with_human_readable_string(self): + with open("foo.py", "w") as f: + f.write('name = input("prompt")') + + process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_human_readable_py") + process.expect_exact(":)") + process.expect_exact("foo.py exists") + process.expect_exact("checking that foo.py exists...") + process.expect_exact(":)") + process.expect_exact("takes input") + process.expect_exact("running python3 foo.py...") + process.expect_exact("sending input bbb...") + process.close(force=True) + class TestCompileExit(SimpleBase): compiled_loc = CHECKS_DIRECTORY / "compile_exit" / "__init__.py" @@ -384,5 +437,29 @@ def test_target_failing_dependency(self): self.assertEqual(output["results"][1]["name"], "exists5") +class TestRemoteException(Base): + def test_no_traceback(self): + # Check that bar (part of traceback) is not shown + process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/remote_exception_no_traceback") + self.assertRaises(pexpect.exceptions.EOF, lambda: process.expect("bar")) + + # Check that foo (the message) is shown + process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/remote_exception_no_traceback") + process.expect("foo") + + def test_traceback(self): + process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/remote_exception_traceback") + process.expect("bar") + process.expect("foo") + + +class TestInternalDirectories(Base): + def test_directories_exist(self): + with open("foo.py", "w") as f: + f.write(os.getcwd()) + + process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/internal_directories") + process.expect_exact(":)") + if __name__ == "__main__": unittest.main() diff --git a/tests/checks/internal_directories/.cs50.yaml b/tests/checks/internal_directories/.cs50.yaml new file mode 100644 index 00000000..efa53149 --- /dev/null +++ b/tests/checks/internal_directories/.cs50.yaml @@ -0,0 +1 @@ +check50: true diff --git a/tests/checks/internal_directories/__init__.py b/tests/checks/internal_directories/__init__.py new file mode 100644 index 00000000..c5a74efe --- /dev/null +++ b/tests/checks/internal_directories/__init__.py @@ -0,0 +1,20 @@ +import pathlib +import check50 +import check50.internal as internal + +@check50.check() +def foo(): + """directories exist""" + assert internal.run_dir.resolve() == pathlib.Path.cwd() + assert internal.run_dir.parent == internal.run_root_dir + +assert internal.run_root_dir.exists() +assert internal.run_root_dir.resolve() == pathlib.Path.cwd() + +assert internal.student_dir.exists() +with open(internal.student_dir / "foo.py") as f: + student_dir = pathlib.Path(f.read().strip()) +assert internal.student_dir == student_dir + +assert internal.check_dir.exists() +assert internal.check_dir.name == "internal_directories" diff --git a/tests/checks/remote_exception_no_traceback/.cs50.yaml b/tests/checks/remote_exception_no_traceback/.cs50.yaml new file mode 100644 index 00000000..0745d8da --- /dev/null +++ b/tests/checks/remote_exception_no_traceback/.cs50.yaml @@ -0,0 +1,2 @@ +check50: + checks: check.py diff --git a/tests/checks/remote_exception_no_traceback/check.py b/tests/checks/remote_exception_no_traceback/check.py new file mode 100644 index 00000000..9ce01487 --- /dev/null +++ b/tests/checks/remote_exception_no_traceback/check.py @@ -0,0 +1,22 @@ +import check50 +from check50._exceptions import RemoteCheckError + +json = { + "slug": "jelleas/foo/master", + "error": { + "type": "InvalidSlugError", + "value": "foo", + "traceback": [ + "Traceback (most recent call last):\n", + "bar\n" + ], + "actions": { + "show_traceback": False, + "message": "foo" + }, + "data": {} + }, + "version": "3.1.1" +} + +raise RemoteCheckError(json) diff --git a/tests/checks/remote_exception_traceback/.cs50.yaml b/tests/checks/remote_exception_traceback/.cs50.yaml new file mode 100644 index 00000000..0745d8da --- /dev/null +++ b/tests/checks/remote_exception_traceback/.cs50.yaml @@ -0,0 +1,2 @@ +check50: + checks: check.py diff --git a/tests/checks/remote_exception_traceback/check.py b/tests/checks/remote_exception_traceback/check.py new file mode 100644 index 00000000..85c5d8f9 --- /dev/null +++ b/tests/checks/remote_exception_traceback/check.py @@ -0,0 +1,22 @@ +import check50 +from check50._exceptions import RemoteCheckError + +json = { + "slug": "jelleas/foo/master", + "error": { + "type": "InvalidSlugError", + "value": "foo", + "traceback": [ + "Traceback (most recent call last):\n", + "bar\n" + ], + "actions": { + "show_traceback": True, + "message": "foo" + }, + "data": {} + }, + "version": "3.1.1" +} + +raise RemoteCheckError(json) diff --git a/tests/checks/stdin_human_readable_py/.cs50.yaml b/tests/checks/stdin_human_readable_py/.cs50.yaml new file mode 100644 index 00000000..0745d8da --- /dev/null +++ b/tests/checks/stdin_human_readable_py/.cs50.yaml @@ -0,0 +1,2 @@ +check50: + checks: check.py diff --git a/tests/checks/stdin_human_readable_py/check.py b/tests/checks/stdin_human_readable_py/check.py new file mode 100644 index 00000000..0e44e8ca --- /dev/null +++ b/tests/checks/stdin_human_readable_py/check.py @@ -0,0 +1,11 @@ +import check50 + +@check50.check() +def exists(): + """foo.py exists""" + check50.exists("foo.py") + +@check50.check(exists) +def takes_input(): + """takes input""" + check50.run("python3 foo.py").stdin("aaa", prompt=False, str_line="bbb") diff --git a/tests/checks/stdout_py/check.py b/tests/checks/stdout_py/check.py index 9a47dea2..bd9666a6 100644 --- a/tests/checks/stdout_py/check.py +++ b/tests/checks/stdout_py/check.py @@ -8,4 +8,4 @@ def exists(): @check50.check(exists) def prints_hello(): """prints hello""" - check50.run("python3 foo.py").stdout("hello") + check50.run("python3 foo.py").stdout("hello", show_timeout=True, timeout=1)