diff --git a/xicam/plugins/__init__.py b/xicam/plugins/__init__.py index b839806..ba64859 100644 --- a/xicam/plugins/__init__.py +++ b/xicam/plugins/__init__.py @@ -79,7 +79,10 @@ def __init__(self): # Places to look for plugins self.plugindirs = [user_plugin_dir, site_plugin_dir] \ - + list(xicam.__path__) + + list(xicam.__path__) + if getattr(sys,'frozen',False): + self.plugindirs.append(os.path.join(os.path.dirname(sys.argv[0]),'_plugins')) + self.setPluginPlaces(self.plugindirs) msg.logMessage('plugindirectories:', *self.plugindirs) @@ -150,7 +153,10 @@ def collectPlugins(self, paths=None): Overloaded to add callback. """ - self.setPluginPlaces(self.plugindirs + (paths or [])) + dirs = self.plugindirs + if venvs.current_environment: + dirs.append(venvs.current_environment) + self.setPluginPlaces(dirs) self.locatePlugins() @@ -205,7 +211,6 @@ def showLoading(self, plugininfo: PluginInfo): msg.logMessage(f'Loading {name} from {plugininfo.path}') def venvChanged(self): - self.setPluginPlaces([venvs.current_environment]) self.collectPlugins() def __getitem__(self, item: str): @@ -242,6 +247,9 @@ def loadPlugins(self, callback=None): initial_len = len(self.loadqueue) + if not len(self.loadqueue): + return [] + for candidate_infofile, candidate_filepath, plugin_info in iter(self.loadqueue.popleft, (None, None, None)): # if a callback exists, call it before attempting to load # the plugin so that a message can be displayed to the diff --git a/xicam/plugins/cammart.py b/xicam/plugins/cammart.py index aff1397..c853fe9 100644 --- a/xicam/plugins/cammart.py +++ b/xicam/plugins/cammart.py @@ -2,33 +2,64 @@ import json from urllib import parse import os, sys - +import pkg_resources import requests import yaml from appdirs import user_config_dir, site_config_dir, user_cache_dir import platform import subprocess +from xicam.core import msg +from importlib import reload +import pip + +from . import manager +from . import venvs def pipmain(args): - subprocess.call([sys.executable, "-m", "pip", *args]) + old_env = os.environ.copy() + os.environ["PYTHONPATH"] = os.path.join( + venvs.current_environment, "Lib/site-packages" + ) + os.environ["PYTHONBASE"] = venvs.current_environment + old_prefix = sys.prefix + sys.prefix = venvs.current_environment + + r = 1 # (error code) + try: + # Install the bundled software + try: + r = pip.main(args) + except AttributeError: + from pip._internal import main -# try: -# from pip._internal import main as pipmain -# except ModuleNotFoundError: -# from pip import main as pipmain + r = main(args) + except Exception as ex: + msg.logError(ex) -from . import manager -from . import venvs + finally: + os.environ = old_env + sys.prefix = old_prefix + return r + + # subprocess.call([os.path.join(venvs.current_environment, 'Scripts/pip.exe'), *args], env=env) + # return 0 # Above subprocess typicall returns error code even though it succeeds op_sys = platform.system() -if op_sys == 'Darwin': # User config dir incompatible with venv on darwin (space in path name conflicts) - user_package_registry = os.path.join(user_cache_dir(appname='xicam'),'packages.yml') +if ( + op_sys == "Darwin" +): # User config dir incompatible with venv on darwin (space in path name conflicts) + user_package_registry = os.path.join( + user_cache_dir(appname="xicam"), "packages.yml" + ) else: - user_package_registry = os.path.join(user_config_dir(appname='xicam'),'packages.yml') -site_package_registry = os.path.join(site_config_dir(appname='xicam'),'packages.yml') + user_package_registry = os.path.join( + user_config_dir(appname="xicam"), "packages.yml" + ) +site_package_registry = os.path.join(site_config_dir(appname="xicam"), "packages.yml") + def install(name: str): """ @@ -45,21 +76,42 @@ def install(name: str): # TODO: check if package is in repo # Get install plugin package information from cam-mart repository - o = requests.get(f'http://cam.lbl.gov:5000/pluginpackages?where={{"name":"{name}"}}') + o = requests.get( + f'http://cam.lbl.gov:5000/pluginpackages?where={{"name":"{name}"}}' + ) # Get the uri from the plugin package information - uri = parse.urlparse(json.loads(o.content)['_items'][0]["installuri"]) + uri = parse.urlparse(json.loads(o.content)["_items"][0]["installuri"]) failure = True - print('Installing to:', venvs.current_environment) + # import setuptools + + # print("Updating setuptools...") + # print("version:", setuptools.__version__) + # pipmain(["install", "--upgrade", "--ignore-installed", "setuptools"]) + # setuptools = reload(setuptools) + + print("Installing to:", venvs.current_environment) # Install from the uri - if uri.scheme == 'pipgit': # Clones a git repo and installs with pip - failure = pipmain(['install', f'--target={venvs.current_environment}', 'git+https://' + ''.join(uri[1:])]) - elif uri.scheme == 'pip': - failure = pipmain(['install', f'--target={venvs.current_environment}', ''.join(uri[1:])]) - elif uri.scheme == 'conda': + if uri.scheme == "pipgit": # Clones a git repo and installs with pip + failure = pipmain( + [ + "install", + # f'--target={os.path.join(venvs.current_environment,"Lib/site-packages")}', + "git+https://" + "".join(uri[1:]), + ] + ) + elif uri.scheme == "pip": + failure = pipmain( + [ + "install", + # f'--target={os.path.join(venvs.current_environment,"Lib/site-packages")}', + "".join(uri[1:]), + ] + ) + elif uri.scheme == "conda": raise NotImplementedError if not failure: @@ -69,12 +121,15 @@ def install(name: str): def uninstall(name: str): + # Note: pkg_resources keeps a "working_set" that won't normally be updated when venv changes... + # An update may need to be forcefully triggered in the future + pkg_resources._initialize_master_working_set() failure = True if name in pkg_registry: scheme = pkg_registry[name] - if scheme in ['pipgit', 'pip']: - failure = pipmain(['uninstall', '-y', name]) - elif scheme == 'conda': + if scheme in ["pipgit", "pip"]: + failure = pipmain(["uninstall", "-y", name]) + elif scheme == "conda": raise NotImplementedError else: # TODO: Handle if name is not in the registry @@ -85,6 +140,7 @@ def uninstall(name: str): return not failure + class pkg_registry(collections.MutableMapping): def __init__(self): self._store = dict() @@ -114,13 +170,13 @@ def __keytransform__(self, key): def load(self): try: - with open(user_package_registry, 'r') as f: + with open(user_package_registry, "r") as f: self._store = yaml.load(f.read()) except FileNotFoundError: pass def save(self): - with open(user_package_registry, 'w') as f: + with open(user_package_registry, "w") as f: f.write(yaml.dump(self._store)) diff --git a/xicam/plugins/venvs.py b/xicam/plugins/venvs.py index c2b30db..6c34dc6 100644 --- a/xicam/plugins/venvs.py +++ b/xicam/plugins/venvs.py @@ -27,86 +27,6 @@ # TODO: transition to http://virtualenvwrapper.readthedocs.io/en/latest/index.html -class EnvBuilder(venv.EnvBuilder): - def _setup_pip(self, context): - """ Override normal behavior using subprocess, call ensurepip directly""" - self.bootstrap(root=os.path.dirname(os.path.dirname(context.env_exe)), upgrade=True, default_pip=True) - - @staticmethod - def bootstrap(*, root=None, upgrade=False, user=False, - altinstall=False, default_pip=False, - verbosity=0): - """ - Bootstrap modified from ensurepip to avoid issues with parent environment - """ - if altinstall and default_pip: - raise ValueError("Cannot use altinstall and default_pip together") - - ensurepip._disable_pip_configuration_settings() - - # By default, installing pip and setuptools installs all of the - # following scripts (X.Y == running Python version): - # - # pip, pipX, pipX.Y, easy_install, easy_install-X.Y - # - # pip 1.5+ allows ensurepip to request that some of those be left out - if altinstall: - # omit pip, pipX and easy_install - os.environ["ENSUREPIP_OPTIONS"] = "altinstall" - elif not default_pip: - # omit pip and easy_install - os.environ["ENSUREPIP_OPTIONS"] = "install" - - with tempfile.TemporaryDirectory() as tmpdir: - # Put our bundled wheels into a temporary directory and construct the - # additional paths that need added to sys.path - additional_paths = [] - for project, version in ensurepip._PROJECTS: - wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) - whl = pkgutil.get_data( - "ensurepip", - "_bundled/{}".format(wheel_name), - ) - with open(os.path.join(tmpdir, wheel_name), "wb") as fp: - fp.write(whl) - - additional_paths.append(os.path.join(tmpdir, wheel_name)) - - # Construct the arguments to be passed to the pip command - args = ["install", "--no-index", "--find-links", tmpdir] - if root: - args += ["--prefix", root] ######## Modified - if upgrade: - args += ["--upgrade"] - if user: - args += ["--user"] - if verbosity: - args += ["-" + "v" * verbosity] - - args += ['--ignore-installed'] ######## Added - - print('boostrap:', *(args + [p[0] for p in ensurepip._PROJECTS])) - EnvBuilder._run_pip(args + [p[0] for p in ensurepip._PROJECTS], additional_paths) - - @staticmethod - def _run_pip(args, additional_paths=None): - oldpath=sys.path - - # Add our bundled software to the sys.path so we can import it - if additional_paths is not None: - sys.path = additional_paths + sys.path - - # Install the bundled software - import pip - try: - pip.main(args) - except AttributeError: - from pip._internal import main - main(args) - - # Restore sys.path - sys.path=oldpath - def create_environment(name: str): """ @@ -122,7 +42,7 @@ def create_environment(name: str): if envpath.is_dir(): return # raise ValueError('Environment already exists.') - EnvBuilder(with_pip=True).create(envpath) + venv.EnvBuilder(with_pip=False).create(envpath) def use_environment(name): @@ -150,18 +70,103 @@ def activate_this(path): # Below copied and modified from the activate_this.py module of virtualenv, which is missing form venv old_os_path = os.environ.get('PATH', '') os.environ['PATH'] = os.path.dirname(os.path.abspath(path)) + os.pathsep + old_os_path - base = os.path.dirname(os.path.dirname(os.path.abspath(path))) if sys.platform == 'win32': - site_packages = os.path.join(base, 'Lib', 'site-packages') + site_packages = os.path.join(path, 'Lib', 'site-packages') else: - site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') + site_packages = os.path.join(path, 'lib', 'python%s' % sys.version[:3], 'site-packages') prev_sys_path = list(sys.path) # if not getattr(sys, 'frozen', False): # site missing addsitedir when frozen - import site - site.addsitedir(site_packages) + try: + import site + addsitedir = site.addsitedir + except AttributeError: # frozen apps have no site-packages; and thus their site.py is fake + # NOTE: relevant methods have been extracted from a real site.py + def makepath(*paths): + dir = os.path.join(*paths) + try: + dir = os.path.abspath(dir) + except OSError: + pass + return dir, os.path.normcase(dir) + def _init_pathinfo(): + """Return a set containing all existing file system items from sys.path.""" + d = set() + for item in sys.path: + try: + if os.path.exists(item): + _, itemcase = makepath(item) + d.add(itemcase) + except TypeError: + continue + return d + + def addpackage(sitedir, name, known_paths): + """Process a .pth file within the site-packages directory: + For each line in the file, either combine it with sitedir to a path + and add that to known_paths, or execute it if it starts with 'import '. + """ + if known_paths is None: + known_paths = _init_pathinfo() + reset = True + else: + reset = False + fullname = os.path.join(sitedir, name) + try: + f = open(fullname, "r") + except OSError: + return + with f: + for n, line in enumerate(f): + if line.startswith("#"): + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception: + print("Error processing line {:d} of {}:\n".format(n + 1, fullname), + file=sys.stderr) + import traceback + for record in traceback.format_exception(*sys.exc_info()): + for line in record.splitlines(): + print(' ' + line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break + if reset: + known_paths = None + return known_paths + def addsitedir(sitedir, known_paths=None): + """Add 'sitedir' argument to sys.path if missing and handle .pth files in + 'sitedir'""" + if known_paths is None: + known_paths = _init_pathinfo() + reset = True + else: + reset = False + sitedir, sitedircase = makepath(sitedir) + if not sitedircase in known_paths: + sys.path.append(sitedir) # Add path component + known_paths.add(sitedircase) + try: + names = os.listdir(sitedir) + except OSError: + return + names = [name for name in names if name.endswith(".pth")] + for name in sorted(names): + addpackage(sitedir, name, known_paths) + if reset: + known_paths = None + return known_paths + + addsitedir(site_packages) sys.real_prefix = sys.prefix - sys.prefix = base + sys.prefix = path # Move the added items to the front of the path: new_sys_path = [] for item in list(sys.path):