From 21daf2639d8caf652d35c03be80c3e27f5f2f17c Mon Sep 17 00:00:00 2001 From: Jan Kieseler Date: Sat, 28 Mar 2020 15:30:00 +0100 Subject: [PATCH] python3 migration, will tag as 3.0 --- docker/Dockerfile_base | 9 +++++- environment/activateROOT.sh | 23 --------------- environment/djcenv.conda | 59 ------------------------------------- environment/djcenv.pip | 42 -------------------------- environment/djcenv_gpu.pip | 42 -------------------------- environment/setup_djcenv.sh | 48 ------------------------------ modeltools.py | 32 ++------------------ 7 files changed, 11 insertions(+), 244 deletions(-) delete mode 100644 environment/activateROOT.sh delete mode 100644 environment/djcenv.conda delete mode 100644 environment/djcenv.pip delete mode 100644 environment/djcenv_gpu.pip delete mode 100755 environment/setup_djcenv.sh diff --git a/docker/Dockerfile_base b/docker/Dockerfile_base index ff8d88f..4a27a4e 100644 --- a/docker/Dockerfile_base +++ b/docker/Dockerfile_base @@ -63,6 +63,13 @@ RUN pip3 install opencv-python easydict tqdm Cython numba RUN pip3 install gpustat setGPU ## torch/jupyter stuff -RUN pip3 install torch future gensim jupyter prompt-toolkit +RUN pip3 install torch future gensim jupyter prompt-toolkit torch_geometric EXPOSE 8888 +#for singularity panic +RUN adduser --disabled-password --gecos "" dummyuser + + + + + diff --git a/environment/activateROOT.sh b/environment/activateROOT.sh deleted file mode 100644 index 9db0bbc..0000000 --- a/environment/activateROOT.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# hack to make root-numpy and ROOT work properly in python -# this is necessary because the original package that is checked -# out by conda is broken - -cd ${CONDA_PREFIX} -source bin/thisroot.sh -cd - - -echo "Activate: ROOT has been sourced. Environment settings are ready. " -echo "ROOTSYS="${ROOTSYS} - -if [ -n "${LD_LIBRARY_PATH}" ]; then - unset LD_LIBRARY_PATH -fi - - -if [ -n "${DYLD_LIBRARY_PATH}" ]; then - unset DYLD_LIBRARY_PATH -fi - - diff --git a/environment/djcenv.conda b/environment/djcenv.conda deleted file mode 100644 index 31ff785..0000000 --- a/environment/djcenv.conda +++ /dev/null @@ -1,59 +0,0 @@ -# This file may be used to create an environment using: -# $ conda create --name --file -# platform: linux-64 -@EXPLICIT -https://repo.continuum.io/pkgs/main/linux-64/ca-certificates-2019.1.23-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/libgfortran-3.0.0-1.tar.bz2 -https://repo.continuum.io/pkgs/main/linux-64/libstdcxx-ng-8.2.0-hdf63c60_1.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/expat-2.1.0-0.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/fftw-3.3.4-2.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/glibc-2.12.2-3.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/gmp-5.1.2-3.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/gsl-1.16-2.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/icu-54.1-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/jbig-2.1-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/jpeg-9b-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/libffi-3.2.1-1.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/libgcc-5.2.0-0.tar.bz2 -https://repo.continuum.io/pkgs/main/linux-64/libgcc-ng-8.2.0-hdf63c60_1.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/libiconv-1.14-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/libxcb-1.12-1.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/linux-headers-2.6.32-1.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/mkl-2017.0.1-0.tar.bz2 -https://repo.continuum.io/pkgs/main/linux-64/ncurses-6.0-h9df7e31_2.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/openssl-1.0.1k-1.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/pcre-8.37-5.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/readline-6.2-2.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/sqlite-3.13.0-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/system-5.8-2.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/tk-8.5.18-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/xz-5.2.2-1.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/zlib-1.2.8-3.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/dbus-1.10.10-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/glib-2.43.0-1.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/hdf5-1.8.17-1.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/isl-0.12.2-2.tar.bz2 -https://repo.continuum.io/pkgs/main/linux-64/libedit-3.1-heed3624_0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/libpng-1.6.27-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/libtiff-4.0.6-3.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/libxml2-2.9.4-0.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/mpfr-3.1.2-2.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/python-2.7.5-3.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/boost-1.61.0-py27_0.tar.bz2 -https://repo.continuum.io/pkgs/main/linux-64/certifi-2019.3.9-py27_0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/cloog-0.18.0-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/freetype-2.5.5-2.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/gstreamer-1.8.0-0.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/mpc-1.0.1-2.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/numexpr-2.6.2-np113py27_0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/scikit-learn-0.18.2-np113py27_0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/wheel-0.29.0-py27_0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/fontconfig-2.12.1-3.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/gcc-4.8.2-25.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/graphviz-2.38.0-4.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/gst-plugins-base-1.8.0-0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/pytables-3.3.0-np113py27_0.tar.bz2 -https://repo.continuum.io/pkgs/main/linux-64/setuptools-41.0.1-py27_0.tar.bz2 -https://repo.continuum.io/pkgs/free/linux-64/qt-5.6.2-3.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/root-6.04-py2.7_gcc4.8.2.tar.bz2 -https://conda.anaconda.org/NLeSC/linux-64/root-numpy-4.4.0-root6.04_py2.7.tar.bz2 diff --git a/environment/djcenv.pip b/environment/djcenv.pip deleted file mode 100644 index 07635d5..0000000 --- a/environment/djcenv.pip +++ /dev/null @@ -1,42 +0,0 @@ -absl-py==0.7.1 -astor==0.7.1 -backports.weakref==1.0.post1 -bleach==1.5.0 -blessings==1.7 -certifi==2019.3.9 -cycler==0.10.0 -enum34==1.1.6 -funcsigs==1.0.2 -functools32==3.2.3.post2 -futures==3.2.0 -gast==0.2.2 -gpustat==0.5.0 -grpcio==1.20.1 -h5py==2.9.0 -html5lib==0.9999999 -Keras==2.2.4 -Keras-Applications==1.0.7 -Keras-Preprocessing==1.0.9 -Markdown==3.1 -matplotlib==2.0.0 -mock==2.0.0 -numexpr==2.6.2 -numpy==1.16.3 -pbr==2.0.0 -protobuf==3.7.1 -psutil==5.6.2 -pyparsing==2.2.0 -python-dateutil==2.6.0 -pytz==2016.10 -PyYAML==5.1 -root-numpy==4.4.0.dev0 -scikit-learn==0.18.2 -scipy==1.2.1 -six==1.12.0 -subprocess32==3.2.7 -tables==3.3.0 -tensorboard==1.13.1 -tensorflow-estimator==1.13.0 -tensorflow==1.13.1 -termcolor==1.1.0 -Werkzeug==0.15.2 diff --git a/environment/djcenv_gpu.pip b/environment/djcenv_gpu.pip deleted file mode 100644 index 460f2ff..0000000 --- a/environment/djcenv_gpu.pip +++ /dev/null @@ -1,42 +0,0 @@ -absl-py==0.7.1 -astor==0.7.1 -backports.weakref==1.0.post1 -bleach==1.5.0 -blessings==1.7 -cycler==0.10.0 -enum34==1.1.6 -funcsigs==1.0.2 -functools32==3.2.3.post2 -futures==3.2.0 -gast==0.2.2 -gpustat==0.5.0 -grpcio==1.20.1 -h5py==2.9.0 -html5lib==0.9999999 -Keras==2.2.4 -Keras-Applications==1.0.7 -Keras-Preprocessing==1.0.9 -Markdown==3.1 -matplotlib==2.0.0 -mock==2.0.0 -numexpr==2.6.2 -numpy==1.16.3 -pbr==2.0.0 -protobuf==3.7.1 -psutil==5.6.2 -pyparsing==2.2.0 -python-dateutil==2.6.0 -pytz==2016.10 -PyYAML==5.1 -root-numpy==4.4.0.dev0 -scikit-learn==0.18.2 -scipy==1.2.1 -setGPU==0.0.7 -six==1.12.0 -subprocess32==3.2.7 -tables==3.3.0 -tensorboard==1.13.1 -tensorflow-estimator==1.13.0 -tensorflow-gpu==1.13.1 -termcolor==1.1.0 -Werkzeug==0.15.2 diff --git a/environment/setup_djcenv.sh b/environment/setup_djcenv.sh deleted file mode 100755 index 1f2f789..0000000 --- a/environment/setup_djcenv.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -if [ ! `which conda` ] -then -echo Please install the anaconda package manager -exit 1 -fi - - -addstring="" - -if [[ $1 == "gpu" ]] -then - echo "setting up for gpu usage" - addstring="_${1}" -fi - - - -envfile=djcenv.conda -envname="${envfile%.*}${addstring}" -pipfile="${envfile%.*}.pip" -pipfilegpu="${envfile%.*}_gpu.pip" - - -conda create --copy --name $envname python=2.7.13 -conda install --name $envname --file $envfile - -source activate $envname - -if [ $addstring ] -then - pip install -r $pipfilegpu - pip install setGPU -else - pip install -r $pipfile -fi - - -cp activateROOT.sh $CONDA_PREFIX/etc/conda/activate.d/activateROOT.sh - -echo "please ignore the warning that come with the following installation" - -conda install -c menpo ffmpeg - -echo "stop ignoring the warnings again" - -echo "environment set up. Please activate it with \"source activate ${envname}\"" - diff --git a/modeltools.py b/modeltools.py index 820df02..bbba4f4 100644 --- a/modeltools.py +++ b/modeltools.py @@ -1,31 +1,6 @@ -import imp -try: - imp.find_module('Losses') - from Losses import * -except ImportError: - print 'No Losses module found, ignoring at your own risk' - global_loss_list = {} - -try: - imp.find_module('Layers') - from Layers import * -except ImportError: - print 'No Layers module found, ignoring at your own risk' - global_layers_list = {} - -try: - imp.find_module('Metrics') - from Metrics import * -except ImportError: - print 'No metrics module found, ignoring at your own risk' - global_metrics_list = {} - - - -custom_objs = {} -custom_objs.update(global_loss_list) -custom_objs.update(global_layers_list) -custom_objs.update(global_metrics_list) +from DeepJetCore.customObjects import * + +custom_objs = get_custom_objects() def getLayer(model, name): @@ -35,7 +10,6 @@ def getLayer(model, name): - def printLayerInfosAndWeights(model, noweights=False): for layer in model.layers: g=layer.get_config()