diff --git a/doc/source/api/install.rst b/doc/source/api/install.rst index 1dd0e442e..3e5c9c5dd 100644 --- a/doc/source/api/install.rst +++ b/doc/source/api/install.rst @@ -15,6 +15,7 @@ Subpackages install.1_2_install install.2_0_install install.2_1_install + install.2_3_1_install install.2_3_install install.future_install diff --git a/doc/source/api_plugin/install.rst b/doc/source/api_plugin/install.rst index 3a6ed134e..58d356632 100644 --- a/doc/source/api_plugin/install.rst +++ b/doc/source/api_plugin/install.rst @@ -9,6 +9,7 @@ Subpackages install.1_2_install install.2_0_install install.2_1_install + install.2_3_1_install install.2_3_install install.future_install diff --git a/doc/source/dls_installer.rst b/doc/source/dls_installer.rst index 468af8b92..09acdefbf 100644 --- a/doc/source/dls_installer.rst +++ b/doc/source/dls_installer.rst @@ -4,9 +4,9 @@ Installing Savu Requirements: - An MPI implementation (tested with openmpi 2.0.1) - fftw (tested with 3.3.7) - - cuda (tested with 9.1) + - cuda (tested with 8.0) -1. Download the latest version of :download:`savu <../../install/2_3_install/savu_v2.3.1.tar.gz>` and extract. +1. Download the latest version of :download:`savu <../../install/2_3_1_install/savu_v2.3.1.tar.gz>` and extract. 2. Run the following command and follow the installation instructions: diff --git a/doc/source/framework_autosummary.rst b/doc/source/framework_autosummary.rst index acd6cf625..af45e5936 100644 --- a/doc/source/framework_autosummary.rst +++ b/doc/source/framework_autosummary.rst @@ -14,43 +14,32 @@ savu.core ------------------------------------------------------------ .. toctree:: - api/savu.core.utils - api/savu.core.plugin_runner api/savu.core.basic_plugin_runner api/savu.core.checkpointing + api/savu.core.plugin_runner api/savu.core.transport_setup + api/savu.core.utils savu.core.transports ------------------------------------------------------------ .. toctree:: - api/savu.core.transports.hdf5_transport api/savu.core.transports.base_transport api/savu.core.transports.basic_transport api/savu.core.transports.dosna_transport + api/savu.core.transports.hdf5_transport savu.data ------------------------------------------------------------ .. toctree:: - api/savu.data.experiment_collection - api/savu.data.plugin_list api/savu.data.chunking - api/savu.data.meta_data + api/savu.data.experiment_collection api/savu.data.framework_citations - - -savu.data.transport_data ------------------------------------------------------------- - -.. toctree:: - api/savu.data.transport_data.hdf5_transport_data - api/savu.data.transport_data.base_transport_data - api/savu.data.transport_data.basic_transport_data - api/savu.data.transport_data.dosna_transport_data - api/savu.data.transport_data.slice_lists + api/savu.data.meta_data + api/savu.data.plugin_list savu.data.data_structures @@ -70,8 +59,8 @@ savu.data.data_structures.data_types ------------------------------------------------------------ .. toctree:: - api/savu.data.data_structures.data_types.base_type api/savu.data.data_structures.data_types.data_plus_darks_and_flats + api/savu.data.data_structures.data_types.base_type api/savu.data.data_structures.data_types.fabIO api/savu.data.data_structures.data_types.map_3dto4d_h5 api/savu.data.data_structures.data_types.mrc @@ -79,3 +68,14 @@ savu.data.data_structures.data_types api/savu.data.data_structures.data_types.stitch_data +savu.data.transport_data +------------------------------------------------------------ + +.. toctree:: + api/savu.data.transport_data.base_transport_data + api/savu.data.transport_data.basic_transport_data + api/savu.data.transport_data.dosna_transport_data + api/savu.data.transport_data.hdf5_transport_data + api/savu.data.transport_data.slice_lists + + diff --git a/doc/source/plugin_autosummary.rst b/doc/source/plugin_autosummary.rst index b952eb232..001d509ba 100644 --- a/doc/source/plugin_autosummary.rst +++ b/doc/source/plugin_autosummary.rst @@ -14,43 +14,32 @@ savu.core ------------------------------------------------------------ .. toctree:: - api_plugin/savu.core.utils - api_plugin/savu.core.plugin_runner api_plugin/savu.core.basic_plugin_runner api_plugin/savu.core.checkpointing + api_plugin/savu.core.plugin_runner api_plugin/savu.core.transport_setup + api_plugin/savu.core.utils savu.core.transports ------------------------------------------------------------ .. toctree:: - api_plugin/savu.core.transports.hdf5_transport api_plugin/savu.core.transports.base_transport api_plugin/savu.core.transports.basic_transport api_plugin/savu.core.transports.dosna_transport + api_plugin/savu.core.transports.hdf5_transport savu.data ------------------------------------------------------------ .. toctree:: - api_plugin/savu.data.experiment_collection - api_plugin/savu.data.plugin_list api_plugin/savu.data.chunking - api_plugin/savu.data.meta_data + api_plugin/savu.data.experiment_collection api_plugin/savu.data.framework_citations - - -savu.data.transport_data ------------------------------------------------------------- - -.. toctree:: - api_plugin/savu.data.transport_data.hdf5_transport_data - api_plugin/savu.data.transport_data.base_transport_data - api_plugin/savu.data.transport_data.basic_transport_data - api_plugin/savu.data.transport_data.dosna_transport_data - api_plugin/savu.data.transport_data.slice_lists + api_plugin/savu.data.meta_data + api_plugin/savu.data.plugin_list savu.data.data_structures @@ -70,8 +59,8 @@ savu.data.data_structures.data_types ------------------------------------------------------------ .. toctree:: - api_plugin/savu.data.data_structures.data_types.base_type api_plugin/savu.data.data_structures.data_types.data_plus_darks_and_flats + api_plugin/savu.data.data_structures.data_types.base_type api_plugin/savu.data.data_structures.data_types.fabIO api_plugin/savu.data.data_structures.data_types.map_3dto4d_h5 api_plugin/savu.data.data_structures.data_types.mrc @@ -79,43 +68,64 @@ savu.data.data_structures.data_types api_plugin/savu.data.data_structures.data_types.stitch_data +savu.data.transport_data +------------------------------------------------------------ + +.. toctree:: + api_plugin/savu.data.transport_data.base_transport_data + api_plugin/savu.data.transport_data.basic_transport_data + api_plugin/savu.data.transport_data.dosna_transport_data + api_plugin/savu.data.transport_data.hdf5_transport_data + api_plugin/savu.data.transport_data.slice_lists + + savu.plugins ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.utils + api_plugin/savu.plugins.docstring_parser api_plugin/savu.plugins.plugin api_plugin/savu.plugins.plugin_datasets - api_plugin/savu.plugins.docstring_parser api_plugin/savu.plugins.plugin_datasets_notes + api_plugin/savu.plugins.utils -savu.plugins.azimuthal_integrators +savu.plugins.absorption_corrections ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.azimuthal_integrators.base_azimuthal_integrator - api_plugin/savu.plugins.azimuthal_integrators.pyfai_azimuthal_integrator - api_plugin/savu.plugins.azimuthal_integrators.pyfai_azimuthal_integrator_separate - api_plugin/savu.plugins.azimuthal_integrators.pyfai_azimuthal_integrator_with_bragg_filter + api_plugin/savu.plugins.absorption_corrections.base_absorption_correction + api_plugin/savu.plugins.absorption_corrections.mc_near_absorption_correction -savu.plugins.absorption_corrections +savu.plugins.alignment ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.absorption_corrections.base_absorption_correction - api_plugin/savu.plugins.absorption_corrections.mc_near_absorption_correction + api_plugin/savu.plugins.alignment.projection_vertical_alignment + api_plugin/savu.plugins.alignment.projection_shift + api_plugin/savu.plugins.alignment.sinogram_alignment + api_plugin/savu.plugins.alignment.sinogram_clean -savu.plugins.ptychography +savu.plugins.analysis ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.ptychography.base_ptycho - api_plugin/savu.plugins.ptychography.dummy_ptycho - api_plugin/savu.plugins.ptychography.ptypy_batch - api_plugin/savu.plugins.ptychography.ptypy_compact + api_plugin/savu.plugins.analysis.base_analysis + api_plugin/savu.plugins.analysis.histogram + api_plugin/savu.plugins.analysis.stats + api_plugin/savu.plugins.analysis.stxm_analysis + + +savu.plugins.azimuthal_integrators +------------------------------------------------------------ + +.. toctree:: + api_plugin/savu.plugins.azimuthal_integrators.base_azimuthal_integrator + api_plugin/savu.plugins.azimuthal_integrators.pyfai_azimuthal_integrator_with_bragg_filter + api_plugin/savu.plugins.azimuthal_integrators.pyfai_azimuthal_integrator + api_plugin/savu.plugins.azimuthal_integrators.pyfai_azimuthal_integrator_separate savu.plugins.basic_operations @@ -130,10 +140,10 @@ savu.plugins.centering ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.centering.vo_centering - api_plugin/savu.plugins.centering.vo_centering_new api_plugin/savu.plugins.centering.vo_centering_iterative + api_plugin/savu.plugins.centering.vo_centering api_plugin/savu.plugins.centering.vo_centering_gpu + api_plugin/savu.plugins.centering.vo_centering_new savu.plugins.component_analysis @@ -145,129 +155,88 @@ savu.plugins.component_analysis api_plugin/savu.plugins.component_analysis.pca -savu.plugins.fluo_fitters ------------------------------------------------------------- - -.. toctree:: - api_plugin/savu.plugins.fluo_fitters.base_fluo_fitter - api_plugin/savu.plugins.fluo_fitters.fastxrf_fitting - api_plugin/savu.plugins.fluo_fitters.simple_fit_xrf - - -savu.plugins.analysis ------------------------------------------------------------- - -.. toctree:: - api_plugin/savu.plugins.analysis.base_analysis - api_plugin/savu.plugins.analysis.histogram - api_plugin/savu.plugins.analysis.stats - api_plugin/savu.plugins.analysis.stxm_analysis - - -savu.plugins.reconstructions ------------------------------------------------------------- - -.. toctree:: - api_plugin/savu.plugins.reconstructions.base_recon - api_plugin/savu.plugins.reconstructions.scikitimage_sart - api_plugin/savu.plugins.reconstructions.simple_recon - api_plugin/savu.plugins.reconstructions.scikitimage_filter_back_projection - api_plugin/savu.plugins.reconstructions.ccpi_cgls_recon - api_plugin/savu.plugins.reconstructions.non_local_regularised_cgls - api_plugin/savu.plugins.reconstructions.tomopy_recon - - -savu.plugins.reconstructions.astra_recons ------------------------------------------------------------- - -.. toctree:: - api_plugin/savu.plugins.reconstructions.astra_recons.astra_recon_cpu - api_plugin/savu.plugins.reconstructions.astra_recons.astra_recon_gpu - api_plugin/savu.plugins.reconstructions.astra_recons.base_astra_recon - - -savu.plugins.developing ------------------------------------------------------------- - -.. toctree:: - api_plugin/savu.plugins.developing.testing_sino_align - - savu.plugins.corrections ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.corrections.dark_flat_field_correction - api_plugin/savu.plugins.corrections.timeseries_field_corrections - api_plugin/savu.plugins.corrections.time_based_correction - api_plugin/savu.plugins.corrections.time_based_plus_drift_correction api_plugin/savu.plugins.corrections.base_correction api_plugin/savu.plugins.corrections.camera_rot_correction + api_plugin/savu.plugins.corrections.dark_flat_field_correction + api_plugin/savu.plugins.corrections.convert_360_180_sinogram api_plugin/savu.plugins.corrections.distortion_correction + api_plugin/savu.plugins.corrections.time_based_correction api_plugin/savu.plugins.corrections.monitor_correction api_plugin/savu.plugins.corrections.monitor_correction_nd api_plugin/savu.plugins.corrections.subpixel_shift + api_plugin/savu.plugins.corrections.time_based_plus_drift_correction + api_plugin/savu.plugins.corrections.timeseries_field_corrections api_plugin/savu.plugins.corrections.xrd_absorption_approximation - api_plugin/savu.plugins.corrections.convert_360_180_sinogram - - -savu.plugins.fitters ------------------------------------------------------------- - -.. toctree:: - api_plugin/savu.plugins.fitters.base_fitter - api_plugin/savu.plugins.fitters.ral_fit - api_plugin/savu.plugins.fitters.reproduce_fit - api_plugin/savu.plugins.fitters.simple_fit -savu.plugins.reshape +savu.plugins.developing ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.reshape.data_removal - api_plugin/savu.plugins.reshape.downsample_filter + api_plugin/savu.plugins.developing.testing_sino_align savu.plugins.driver ------------------------------------------------------------ .. toctree:: + api_plugin/savu.plugins.driver.all_cpus_plugin + api_plugin/savu.plugins.driver.base_driver + api_plugin/savu.plugins.driver.basic_driver api_plugin/savu.plugins.driver.cpu_plugin api_plugin/savu.plugins.driver.gpu_plugin + api_plugin/savu.plugins.driver.iterative_plugin api_plugin/savu.plugins.driver.multi_threaded_plugin api_plugin/savu.plugins.driver.plugin_driver - api_plugin/savu.plugins.driver.all_cpus_plugin api_plugin/savu.plugins.driver.single_node_multi_threaded_plugin - api_plugin/savu.plugins.driver.base_driver - api_plugin/savu.plugins.driver.basic_driver - api_plugin/savu.plugins.driver.iterative_plugin -savu.plugins.ring_removal +savu.plugins.filters ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.ring_removal.ccpi_ring_artefact_filter - api_plugin/savu.plugins.ring_removal.raven_filter - api_plugin/savu.plugins.ring_removal.ring_removal_normalization - api_plugin/savu.plugins.ring_removal.ring_removal_waveletfft + api_plugin/savu.plugins.filters.denoise_bregman_filter + api_plugin/savu.plugins.filters.band_pass + api_plugin/savu.plugins.filters.base_filter + api_plugin/savu.plugins.filters.paganin_filter + api_plugin/savu.plugins.filters.dezinger + api_plugin/savu.plugins.filters.dezinger_simple + api_plugin/savu.plugins.filters.dezinger_sinogram + api_plugin/savu.plugins.filters.dials_find_spots + api_plugin/savu.plugins.filters.find_peaks + api_plugin/savu.plugins.filters.image_interpolation + api_plugin/savu.plugins.filters.list_to_projections + api_plugin/savu.plugins.filters.median_filter + api_plugin/savu.plugins.filters.poly_background_estimator + api_plugin/savu.plugins.filters.pymca + api_plugin/savu.plugins.filters.quantisation_filter + api_plugin/savu.plugins.filters.spectrum_crop + api_plugin/savu.plugins.filters.strip_background + api_plugin/savu.plugins.filters.threshold_filter + api_plugin/savu.plugins.filters.umpa -savu.plugins.alignment +savu.plugins.fitters ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.alignment.projection_shift - api_plugin/savu.plugins.alignment.projection_vertical_alignment - api_plugin/savu.plugins.alignment.sinogram_alignment - api_plugin/savu.plugins.alignment.sinogram_clean + api_plugin/savu.plugins.fitters.base_fitter + api_plugin/savu.plugins.fitters.ral_fit + api_plugin/savu.plugins.fitters.reproduce_fit + api_plugin/savu.plugins.fitters.simple_fit -savu.plugins.missing_data +savu.plugins.fluo_fitters ------------------------------------------------------------ .. toctree:: + api_plugin/savu.plugins.fluo_fitters.base_fluo_fitter + api_plugin/savu.plugins.fluo_fitters.fastxrf_fitting + api_plugin/savu.plugins.fluo_fitters.simple_fit_xrf savu.plugins.kinematics @@ -277,41 +246,16 @@ savu.plugins.kinematics api_plugin/savu.plugins.kinematics.stage_motion -savu.plugins.filters ------------------------------------------------------------- - -.. toctree:: - api_plugin/savu.plugins.filters.band_pass - api_plugin/savu.plugins.filters.denoise_bregman_filter - api_plugin/savu.plugins.filters.dials_find_spots - api_plugin/savu.plugins.filters.base_filter - api_plugin/savu.plugins.filters.find_peaks - api_plugin/savu.plugins.filters.median_filter - api_plugin/savu.plugins.filters.dezinger - api_plugin/savu.plugins.filters.paganin_filter - api_plugin/savu.plugins.filters.poly_background_estimator - api_plugin/savu.plugins.filters.dezinger_simple - api_plugin/savu.plugins.filters.dezinger_sinogram - api_plugin/savu.plugins.filters.image_interpolation - api_plugin/savu.plugins.filters.strip_background - api_plugin/savu.plugins.filters.spectrum_crop - api_plugin/savu.plugins.filters.list_to_projections - api_plugin/savu.plugins.filters.pymca - api_plugin/savu.plugins.filters.quantisation_filter - api_plugin/savu.plugins.filters.threshold_filter - api_plugin/savu.plugins.filters.umpa - - savu.plugins.loaders ------------------------------------------------------------ .. toctree:: api_plugin/savu.plugins.loaders.base_loader - api_plugin/savu.plugins.loaders.multi_savu_loader api_plugin/savu.plugins.loaders.hdf5_template_loader api_plugin/savu.plugins.loaders.image_template_loader - api_plugin/savu.plugins.loaders.random_hdf5_loader api_plugin/savu.plugins.loaders.stitch_data_template_loader + api_plugin/savu.plugins.loaders.multi_savu_loader + api_plugin/savu.plugins.loaders.random_hdf5_loader api_plugin/savu.plugins.loaders.savu_nexus_loader api_plugin/savu.plugins.loaders.yaml_converter @@ -321,20 +265,72 @@ savu.plugins.loaders.full_field_loaders .. toctree:: api_plugin/savu.plugins.loaders.full_field_loaders.dxchange_loader + api_plugin/savu.plugins.loaders.full_field_loaders.fake_nxtomo_loader api_plugin/savu.plugins.loaders.full_field_loaders.image_loader api_plugin/savu.plugins.loaders.full_field_loaders.mrc_loader api_plugin/savu.plugins.loaders.full_field_loaders.multi_nxtomo_loader api_plugin/savu.plugins.loaders.full_field_loaders.nxtomo_loader api_plugin/savu.plugins.loaders.full_field_loaders.random_3d_tomo_loader - api_plugin/savu.plugins.loaders.full_field_loaders.fake_nxtomo_loader -savu.plugins.loaders.utils +savu.plugins.loaders.mapping_loaders ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.loaders.utils.mrc_header - api_plugin/savu.plugins.loaders.utils.yaml_utils + api_plugin/savu.plugins.loaders.mapping_loaders.base_multi_modal_loader + api_plugin/savu.plugins.loaders.mapping_loaders.mm_loader + api_plugin/savu.plugins.loaders.mapping_loaders.nxfluo_loader + api_plugin/savu.plugins.loaders.mapping_loaders.nxmonitor_loader + api_plugin/savu.plugins.loaders.mapping_loaders.nxptycho_loader + api_plugin/savu.plugins.loaders.mapping_loaders.nxstxm_loader + api_plugin/savu.plugins.loaders.mapping_loaders.nxxrd_loader + api_plugin/savu.plugins.loaders.mapping_loaders.p2r_fly_scan_detector_loader + api_plugin/savu.plugins.loaders.mapping_loaders.txm_loader + + +savu.plugins.loaders.mapping_loaders.i08_loaders +------------------------------------------------------------ + +.. toctree:: + api_plugin/savu.plugins.loaders.mapping_loaders.i08_loaders.i08_fluo_loader + + +savu.plugins.loaders.mapping_loaders.i13_loaders +------------------------------------------------------------ + +.. toctree:: + api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_stxm_monitor_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_fluo_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_ptycho_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_speckle_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_stxm_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_stxm_xrf_loader + + +savu.plugins.loaders.mapping_loaders.i14_loaders +------------------------------------------------------------ + +.. toctree:: + api_plugin/savu.plugins.loaders.mapping_loaders.i14_loaders.i14_fluo_loader + + +savu.plugins.loaders.mapping_loaders.i18_loaders +------------------------------------------------------------ + +.. toctree:: + api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.base_i18_multi_modal_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_fluo_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_mm_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_monitor_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_stxm_loader + api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_xrd_loader + + +savu.plugins.loaders.mapping_loaders.i22_loaders +------------------------------------------------------------ + +.. toctree:: + api_plugin/savu.plugins.loaders.mapping_loaders.i22_loaders.i22_tomo_loader savu.plugins.loaders.templates @@ -361,64 +357,68 @@ savu.plugins.loaders.templates.nexus_templates .. toctree:: -savu.plugins.loaders.mapping_loaders +savu.plugins.loaders.utils ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.loaders.mapping_loaders.base_multi_modal_loader - api_plugin/savu.plugins.loaders.mapping_loaders.mm_loader - api_plugin/savu.plugins.loaders.mapping_loaders.nxfluo_loader - api_plugin/savu.plugins.loaders.mapping_loaders.nxmonitor_loader - api_plugin/savu.plugins.loaders.mapping_loaders.nxptycho_loader - api_plugin/savu.plugins.loaders.mapping_loaders.nxstxm_loader - api_plugin/savu.plugins.loaders.mapping_loaders.nxxrd_loader - api_plugin/savu.plugins.loaders.mapping_loaders.p2r_fly_scan_detector_loader - api_plugin/savu.plugins.loaders.mapping_loaders.txm_loader + api_plugin/savu.plugins.loaders.utils.mrc_header + api_plugin/savu.plugins.loaders.utils.yaml_utils -savu.plugins.loaders.mapping_loaders.i13_loaders +savu.plugins.missing_data ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_fluo_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_ptycho_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_speckle_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_stxm_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_stxm_monitor_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i13_loaders.i13_stxm_xrf_loader -savu.plugins.loaders.mapping_loaders.i22_loaders +savu.plugins.ptychography ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.loaders.mapping_loaders.i22_loaders.i22_tomo_loader + api_plugin/savu.plugins.ptychography.base_ptycho + api_plugin/savu.plugins.ptychography.dummy_ptycho + api_plugin/savu.plugins.ptychography.ptypy_batch + api_plugin/savu.plugins.ptychography.ptypy_compact -savu.plugins.loaders.mapping_loaders.i18_loaders +savu.plugins.reconstructions ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.base_i18_multi_modal_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_fluo_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_mm_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_monitor_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_stxm_loader - api_plugin/savu.plugins.loaders.mapping_loaders.i18_loaders.i18_xrd_loader + api_plugin/savu.plugins.reconstructions.non_local_regularised_cgls + api_plugin/savu.plugins.reconstructions.base_recon + api_plugin/savu.plugins.reconstructions.ccpi_cgls_recon + api_plugin/savu.plugins.reconstructions.scikitimage_filter_back_projection + api_plugin/savu.plugins.reconstructions.scikitimage_sart + api_plugin/savu.plugins.reconstructions.simple_recon + api_plugin/savu.plugins.reconstructions.tomopy_recon -savu.plugins.loaders.mapping_loaders.i08_loaders +savu.plugins.reconstructions.astra_recons ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.loaders.mapping_loaders.i08_loaders.i08_fluo_loader + api_plugin/savu.plugins.reconstructions.astra_recons.astra_recon_cpu + api_plugin/savu.plugins.reconstructions.astra_recons.astra_recon_gpu + api_plugin/savu.plugins.reconstructions.astra_recons.base_astra_recon -savu.plugins.loaders.mapping_loaders.i14_loaders +savu.plugins.reshape ------------------------------------------------------------ .. toctree:: - api_plugin/savu.plugins.loaders.mapping_loaders.i14_loaders.i14_fluo_loader + api_plugin/savu.plugins.reshape.data_removal + api_plugin/savu.plugins.reshape.downsample_filter + + +savu.plugins.ring_removal +------------------------------------------------------------ + +.. toctree:: + api_plugin/savu.plugins.ring_removal.ccpi_ring_artefact_filter + api_plugin/savu.plugins.ring_removal.ring_removal_normalization + api_plugin/savu.plugins.ring_removal.raven_filter + api_plugin/savu.plugins.ring_removal.ring_removal_waveletfft savu.plugins.savers diff --git a/install/2_3_1_install/__init__.py b/install/2_3_1_install/__init__.py new file mode 100644 index 000000000..05ec9bfd3 --- /dev/null +++ b/install/2_3_1_install/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2014 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +All the plugin architecture for Savu is contained here + + +.. moduleauthor:: Mark Basham + +""" + diff --git a/install/2_3_1_install/conda-recipes/__init__.py b/install/2_3_1_install/conda-recipes/__init__.py new file mode 100644 index 000000000..05ec9bfd3 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2014 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +All the plugin architecture for Savu is contained here + + +.. moduleauthor:: Mark Basham + +""" + diff --git a/install/2_3_1_install/conda-recipes/astra/astra.pth b/install/2_3_1_install/conda-recipes/astra/astra.pth new file mode 100644 index 000000000..45c4c6127 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/astra/astra.pth @@ -0,0 +1,2 @@ +./astra/python + diff --git a/install/2_3_1_install/conda-recipes/astra/build.sh b/install/2_3_1_install/conda-recipes/astra/build.sh new file mode 100644 index 000000000..acd72a6d6 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/astra/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +nvcc=`command -v nvcc` +cuda=${nvcc%/bin/nvcc} + +savu_path=`command -v savu` +ana_path=${savu_path%/savu} +prefix=${ana_path%/bin}/lib/python2.7/site-packages/astra +export PATH=$ana_path:$PATH + +cd build/linux + +if [ "$cuda" ]; then + ./configure --with-cuda=$cuda --with-python --prefix=$prefix +else + echo "cuda has not been found." + ./configure --with-python --prefix=$prefix +fi + +make -j 4 +make install + diff --git a/install/2_3_1_install/conda-recipes/astra/meta.yaml b/install/2_3_1_install/conda-recipes/astra/meta.yaml new file mode 100644 index 000000000..4a04c0bc4 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/astra/meta.yaml @@ -0,0 +1,25 @@ +{% set version = "1.8" %} + +package: + name: astra-toolbox + version: 1.8 + +source: + fn: astra-{{ version }}.tar.bz2 + url: http://www.astra-toolbox.com/files/astra-{{ version }}/astra-{{ version }}.tar.bz2 + sha256: 44602e0eb0789a557622eae959fc0a067d81396bcb79ec315cb39b59d52322e7 + +requirements: + build: + - python + - numpy + - cython + +build: + number: 2 + +about: + home: http://www.astra-toolbox.com + license: GPLv3 + summary: 'The ASTRA Toolbox is a Python toolbox of high-performance GPU primitives for 2D and 3D tomography.' + diff --git a/install/2_3_1_install/conda-recipes/h5py/build.sh b/install/2_3_1_install/conda-recipes/h5py/build.sh new file mode 100644 index 000000000..055293a33 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/h5py/build.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# set compiler wrapper +mpicc=$(command -v mpicc) +mpi=`dirname $mpicc` + +# set anaconda distribution +ana_path=$(command -v savu) +ana_path=${ana_path%/bin/savu} +hdf5_version=1.10.1 +hdf5_build_no=1 + +export LD_LIBRARY_PATH=$mpi/lib:$mpi/include:$ana_path/lib:$LD_LIBRARY_PATH +export LD_RUN_PATH=$LD_LIBRARY_PATH +export PYTHONPATH=$PYTHONPATH:$ana_path/lib/python2.7/site-packages +export PATH=$PATH:$ana_path/bin/ +export CC=$mpicc + +$PYTHON setup.py configure --hdf5=$ana_path +$PYTHON setup.py configure --hdf5-version=$hdf5_version +$PYTHON setup.py configure --mpi +$PYTHON setup.py build +$PYTHON setup.py install + diff --git a/install/2_3_1_install/conda-recipes/h5py/meta.yaml b/install/2_3_1_install/conda-recipes/h5py/meta.yaml new file mode 100644 index 000000000..0778446ae --- /dev/null +++ b/install/2_3_1_install/conda-recipes/h5py/meta.yaml @@ -0,0 +1,25 @@ +{% set version = "2.7.1" %} # change this + +package: + name: h5py + version: {{ version }} + +source: + fn: h5py-{{ version }}.tar.gz + url: https://github.com/h5py/h5py/archive/{{ version }}.tar.gz +requirements: + build: + - python + - numpy + - hdf5 + - cython + - pkgconfig + - six + +build: + number: 1 + +about: + home: http://www.h5py.org/ + license: New BSD (http://opensource.org/licenses/bsd-license.php) + diff --git a/install/2_3_1_install/conda-recipes/hdf5/build.sh b/install/2_3_1_install/conda-recipes/hdf5/build.sh new file mode 100644 index 000000000..add607820 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/hdf5/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# set compiler wrapper +mpicc=$(command -v mpicc) +mpi=${mpicc%/bin/mpicc} +export LD_LIBRARY_PATH=$mpi:$mpi/lib:$mpi/include:$LD_LIBRARY_PATH +export LD_RUN_PATH=$LD_LIBRARY_PATH + +# check anaconda distribution +ana_path=$(command -v savu) + +CC=$mpicc ./configure --with-zlib --enable-parallel --enable-shared --prefix=$PREFIX +make +make install + +rm -rf $PREFIX/share/hdf5_examples + diff --git a/install/2_3_1_install/conda-recipes/hdf5/meta.yaml b/install/2_3_1_install/conda-recipes/hdf5/meta.yaml new file mode 100644 index 000000000..a12ea27a8 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/hdf5/meta.yaml @@ -0,0 +1,20 @@ +{% set version = "1.10.1" %} +{% set release = "hdf5-1.10" %} +{% set patch = "hdf5-1.10.1" %} + +package: + name: hdf5 + version: {{ version }} + +source: + fn: {{ patch }}.tar.bz2 + url: https://www.hdfgroup.org/ftp/HDF5/releases/{{ release }}/{{ patch }}/src/{{ patch }}.tar.bz2 + md5: d89893c05ee7ea8611b51bb39450d64e + +build: + number: 1 + +about: + home: http://www.hdfgroup.org/HDF5/ + license: BSD-style (http://www.hdfgroup.org/ftp/HDF5/current/src/unpacked/COPYING) + diff --git a/install/2_3_1_install/conda-recipes/xdesign/build.sh b/install/2_3_1_install/conda-recipes/xdesign/build.sh new file mode 100644 index 000000000..b161f631b --- /dev/null +++ b/install/2_3_1_install/conda-recipes/xdesign/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +$PYTHON setup.py install + diff --git a/install/2_3_1_install/conda-recipes/xdesign/meta.yaml b/install/2_3_1_install/conda-recipes/xdesign/meta.yaml new file mode 100644 index 000000000..68693b427 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/xdesign/meta.yaml @@ -0,0 +1,24 @@ +package: + name: xdesign + version: '0.3.0' + +source: + git_url: https://github.com/tomography/xdesign.git + +build: + number: 0 + +requirements: + build: + - python + - setuptools + +about: + home: http://xdesign.readthedocs.io + license: BSD-3 + summary: 'Benchmarking and optimization tools for tomography.' + +# See +# http://docs.continuum.io/conda/build.html for +# more information about meta.yaml + diff --git a/install/2_3_1_install/conda-recipes/xraylib/build.sh b/install/2_3_1_install/conda-recipes/xraylib/build.sh new file mode 100644 index 000000000..8f0acd316 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/xraylib/build.sh @@ -0,0 +1,7 @@ + +./configure --enable-python --disable-perl --disable-java \ + --disable-fortran2003 --disable-lua --prefix=$PREFIX +make +make check +make install + diff --git a/install/2_3_1_install/conda-recipes/xraylib/meta.yaml b/install/2_3_1_install/conda-recipes/xraylib/meta.yaml new file mode 100644 index 000000000..f924056e2 --- /dev/null +++ b/install/2_3_1_install/conda-recipes/xraylib/meta.yaml @@ -0,0 +1,26 @@ +{% set version = "3.3.0" %} + +package: + name: xraylib + version: {{ version }} + +source: + fn: xraylib-{{ version }}.tar.gz + url: https://xraylib.tomschoonjans.eu/xraylib-{{ version }}.tar.gz + sha256: a22a73b8d90eb752b034bab1a4cf6abdd81b8c7dc5020bcb22132d2ee7aacd42 + +build: + number: 0 + +requirements: + build: + - python + - setuptools + - numpy + - scipy >=0.7.0 + - cython + - swig + - autoconf + - libtool + - automake + diff --git a/install/2_3_1_install/savu_v2.3.1.tar.gz b/install/2_3_1_install/savu_v2.3.1.tar.gz new file mode 100644 index 000000000..3868a0591 Binary files /dev/null and b/install/2_3_1_install/savu_v2.3.1.tar.gz differ diff --git a/install/2_3_1_install/savu_v2.3.1/environment.yml b/install/2_3_1_install/savu_v2.3.1/environment.yml new file mode 100644 index 000000000..75377fe1f --- /dev/null +++ b/install/2_3_1_install/savu_v2.3.1/environment.yml @@ -0,0 +1,29 @@ +name: root +channels: +- defaults +dependencies: +- scipy==1.1.0 +- colorama==0.3.9 +- setuptools==39.2.0 +- jinja2==2.10 +- numpy==1.14.5 +- pandas==0.23.3 +- mock==2.0.0 +- PyWavelets==0.5.2 +- scikit-image==0.14.0 +- scikit-learn==0.19.1 +- pytest==3.6.2 +- spyder==3.3.0 +- ipython==5.7.0 +- cython==0.28.3 +- pip: + - fabio==0.4.0 + - pyfftw==0.10.4 + - nvidia-ml-py==7.352.0 + - PeakUtils==1.1.1 + - pymca==5.3.1 + - tifffile==0.4.0 + - pyreadline==2.1 + - gnureadline==6.3.8 + - sphinx_rtd_theme==0.4.1 + diff --git a/install/2_3_1_install/savu_v2.3.1/savu/build.sh b/install/2_3_1_install/savu_v2.3.1/savu/build.sh new file mode 100644 index 000000000..b6fbaff2b --- /dev/null +++ b/install/2_3_1_install/savu_v2.3.1/savu/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +$PYTHON setup.py install --facility $FACILITY # Python command to install the script. + diff --git a/install/2_3_1_install/savu_v2.3.1/savu/meta.yaml b/install/2_3_1_install/savu_v2.3.1/savu/meta.yaml new file mode 100644 index 000000000..131389903 --- /dev/null +++ b/install/2_3_1_install/savu_v2.3.1/savu/meta.yaml @@ -0,0 +1,23 @@ +{% set version = "2.3.1" %} + +package: + name: savu + version: {{ version }} + +source: + url: https://github.com/DiamondLightSource/Savu/archive/v{{ version }}.tar.gz + +requirements: + build: + - python + +build: + preserve_egg_dir: True + script_env: + - FACILITY + +about: + home: https://github.com/diamondlightsource/Savu + license: Apache V2 and GPL V3 + license_file: LICENCE.txt + diff --git a/install/2_3_1_install/savu_v2.3.1/savu_installer.sh b/install/2_3_1_install/savu_v2.3.1/savu_installer.sh new file mode 100644 index 000000000..1e2500cc3 --- /dev/null +++ b/install/2_3_1_install/savu_v2.3.1/savu_installer.sh @@ -0,0 +1,440 @@ +#!/bin/bash -ex + +# change to 'latest' for the latest version +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +versions_file=$DIR/versions_file.txt + +# error log to screen and file +log_temp=`mktemp -d` +error_log=$log_temp/savu_error_log.txt +exec 2> >(tee -ia $error_log) + +oldprompt=$PS1 +newprompt=">>> " +export PS1=$newprompt + +for sig in INT TERM EXIT; do + trap "export PS1=$oldprompt; [[ $sig == EXIT ]] || kill -$sig $$" $sig +done + +PREFIX="${PREFIX:-$HOME}" + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +savu_version=`cat $DIR/version.txt` + +# function for parsing optional arguments +function arg_parse () +{ + flag=$1 + return=$2 + while [[ $# -gt 2 ]] ; do + if [ $3 == $flag ] ; then + eval "$return"=$4 + fi + shift + done +} + +# function for parsing flags without arguments +function flag_parse () +{ + flag=$1 + return=$2 + while [[ $# -gt 2 ]] ; do + if [ $3 == $flag ] ; then + eval "$return"=true + fi + shift + done +} + +# Set the install PREFIX +flag_parse "--PREFIX" prefix "$@" +if [ $prefix ] ; then + PREFIX=true +fi + +# Set the test flag to true if test only +flag_parse "--tests_only" test_flag "$@" +if [ $test_flag ] ; then + test_flag=true +fi + +# Set the prompts flag to false if no prompts are required +flag_parse "--no_prompts" prompts "$@" +if [ $prompts ] ; then + prompts=false +else + prompts=true +fi + +# set the intermediate folder +arg_parse "-f" facility "$@" +if [ ! $facility ] ; then + facility=dls # change this default? +fi + +export FACILITY=$facility + +# set the intermediate folder +arg_parse "-c" conda_folder "$@" +if [ ! $conda_folder ] ; then + conda_folder=Savu_$savu_version +fi + +# set the intermediate folder +arg_parse "-s" savu_recipe "$@" +if [ ! $savu_recipe ] ; then + savu_recipe=savu +elif [ $savu_recipe = 'master' ] ; then + savu_recipe=savu_master +else + echo "Unknown Savu installation version." +fi + +#=========================library checking============================== + +if [ $test_flag ] ; then + echo -e "\n=============================================================" + echo -e " ......Thank you for running the Savu tests......\n" + echo -e "Performing a library check..." +else + echo -e "\n=============================================================" + echo -e " ......Thank you for running the Savu installer......\n" + echo -e "Performing a library check..." + echo -e "\nNB: An MPI implementation is required to build Savu." + echo -e "fftw is required to build Savu." + echo -e "Cuda is desirable for a full range of plugins." + echo -e "\n=============================================================" +fi + +# set compiler wrapper +MPICC=$(command -v mpicc) +if ! [ "$MPICC" ]; then + echo "ERROR: I require mpicc but I can't find it. Check /path/to/mpi_implementation/bin is in your PATH" + exit 1 +else + echo "Using mpicc: " $MPICC +fi + +# check for fftw +CFLAGS="" +LDFLAGS="" +IFS=: +file_base=libfftw?.so +for p in ${LD_LIBRARY_PATH}; do + file_path=${p}/$file_base + if [ "x$p" != "x" -a -e $file_path ]; then + FFTWHOME=${file_path%/lib/libfftw?.so} + CFLAGS="$FFTWHOME/include" + LDFLAGS="$FFTWHOME/lib" + break + fi +done + +if [ "$CFLAGS" ]; then + echo "Using fftw: " $FFTWHOME +else + echo "fftw has not been found." +fi + +# check for cuda +nvcc=`command -v nvcc` +CUDAHOME=${nvcc%/bin/nvcc} +if [ "$CUDAHOME" ]; then + echo "Using cuda: " $CUDAHOME +else + echo "cuda has not been found." +fi + +if [ $test_flag ] && [ $prompts = true ] ; then + + PYTHONHOME=`command -v conda` + PYTHONHOME=${PYTHONHOME%conda} + if [ ! $PYTHONHOME ] ; then + echo -e "No conda environment found in PATH. Try:" + echo -e " >>> source /savu_setup.sh" + echo -e "Aborting the tests." + exit 1 + fi + + echo -e "=============================================================\n" + while true ; do + read -n 1 -p "Are you happy to proceed with the tests? (y/n): " input + if [ "$input" = "y" ]; then + echo -e "\nProceeding with the tests." + break + elif [ "$input" = "n" ]; then + echo -e "\nAborting the tests." + exit 0 + else + echo -e "\nYour input was unknown.\n" + fi + done +elif [ $prompts = true ] ; then + echo -e "=============================================================\n" + while true ; do + read -n 1 -p "Are you happy to proceed with the installation? (y/n): " input + if [ "$input" = "y" ]; then + echo -e "\nYour input was yes" + break + elif [ "$input" = "n" ]; then + echo -e "\nInstallation process terminated." + exit 0 + else + echo -e "\nYour input was unknown.\n" + fi + done + +#=====================installing other packages========================== + + echo -e "\nInstalling Savu in" $PREFIX + read -p ">>> Press ENTER to continue or input a different path: " input + + if [ "$input" != "" ]; then + PREFIX=$input + fi + + while true; do + if [ -d "$PREFIX" ]; then + PREFIX=$PREFIX/$conda_folder/ + break + fi + echo "The path" $PREFIX "is not recognised" + read -p ">>> Please input a different installation path: " input + PREFIX=$input + done + + if [ -d "$PREFIX" ]; then + echo + while true ; do + read -n 1 -p "The folder $PREFIX already exists. Continue? [y/n]" input + if [ "$input" = "y" ]; then + echo -e "\nStarting the installation........" + break + elif [ "$input" = "n" ]; then + echo -e "\nInstallation process terminated." + exit 0 + else + echo -e "\nYour input was unknown.\n\n" + fi + done + else + # create the folder + mkdir -p $PREFIX + fi +else + if [ ! -d "$PREFIX" ] ; then + mkdir -p $PREFIX + fi +fi + +if [ ! $test_flag ] ; then + + echo -e "\nThank you! Installing Savu into" $PREFIX"\n" + + unset IFS + string=`awk '/^miniconda/' $versions_file` + miniconda_version=`echo $string | cut -d " " -f 2` + wget https://repo.continuum.io/miniconda/Miniconda2-$miniconda_version-Linux-x86_64.sh -O $PREFIX/miniconda.sh; + bash $PREFIX/miniconda.sh -b -p $PREFIX/miniconda + PYTHONHOME=$PREFIX/miniconda/bin + export PATH="$PYTHONHOME:$PATH" + + conda install -y -q conda-build conda-env + conda install -y -q conda-verify + conda env update -n root -f $DIR/environment.yml + + echo "Building Savu..." + conda build $DIR/$savu_recipe + savubuild=`conda build $DIR/$savu_recipe --output` + echo "Installing Savu..." + conda install -y -q --use-local $savubuild + + path=$(python -c "import savu; import os; print os.path.abspath(savu.__file__)") + savu_path=${path%/savu/__init__.pyc} + + # get the savu version + install_path=$(python -c "import savu; import savu.version as sv; print sv.__install__") + recipes=$savu_path/$install_path/conda-recipes + + launcher_path=`command -v savu_launcher.sh` + launcher_path=${launcher_path%/savu_launcher.sh} + if [ "$facility" ]; then + cp $savu_path/system_files/$facility/mpi/savu_launcher.sh $launcher_path + cp $savu_path/system_files/$facility/mpi/savu_mpijob.sh $launcher_path + fi + + + + #----------------------------------------------------------------- + echo "Building astra toolbox..." + conda build $recipes/astra + astrabuild=`conda build $recipes/astra --output` + + echo "Installing astra toolbox..." + conda install -y -q --use-local $astrabuild + + site_path=$(python -c "import site; print site.getsitepackages()[0]") + cp $recipes/astra/astra.pth $site_path + astra_lib_path=$site_path/astra/lib + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Building xraylib..." + conda build $recipes/xraylib + xraylibbuild=`conda build $recipes/xraylib --output` + + echo "Installing xraylib..." + conda install -y -q --use-local $xraylibbuild --no-deps + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Installing tomopy..." + # these packages were missing in copied environment + string=`awk '/^tomopy/' $versions_file` + tomopy_version=`echo $string | cut -d " " -f 2` + string=`awk '/^dxchange/' $versions_file` + dxchange_version=`echo $string | cut -d " " -f 2` + conda install -y -q -c dgursoy tomopy==$tomopy_version --no-deps + conda install -y -q -c dgursoy dxchange==$dxchange_version --no-deps + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Installing ccpi-reconstruction" + string=`awk '/^ccpi/' $versions_file` + ccpi_version=`echo $string | cut -d " " -f 2` + conda install -y -q -c ccpi ccpi-reconstruction==$ccpi_version -c conda-forge --no-deps + conda install -y -q -c ccpi cil_reconstruction==$ccpi_version -c conda-forge --no-deps + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Installing pyfai" + string=`awk '/^pyfai/' $versions_file` + pyfai_version=`echo $string | cut -d " " -f 2` + conda install -y -q -c conda-forge pyfai==$pyfai_version --no-deps + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Installing mpi4py..." + pip uninstall -y -q mpi4py || true + string=`awk '/^mpi4py/' $versions_file` + mpi4py_version=`echo $string | cut -d " " -f 2` + env MPICC=$MPICC pip install mpi4py==$mpi4py_version + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Building hdf5..." + conda uninstall -y -q hdf5 || true + conda build $recipes/hdf5 + hdf5build=`conda build $recipes/hdf5 --output` + + echo "Installing hdf5..." + conda install -y -q --use-local $hdf5build + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Building h5py..." + conda uninstall -y -q h5py || true + conda build $recipes/h5py --no-test + h5pybuild=`conda build $recipes/h5py --output` + + echo "Installing h5py..." + conda install -y -q --use-local $h5pybuild + #----------------------------------------------------------------- + + + echo -e "\n\t***************************************************" + echo -e "\t Package installation complete" + echo -e "\t Check $error_log for errors" + echo -e "\t***************************************************\n" + +fi + +if [ ! $test_flag ] ; then + if [ $prompts = true ] ; then + while true; do + read -n 1 -p "Would you like to run the tests? (y/n): " input + if [ "$input" = "y" ]; then + echo -e "\nYour input was yes" + test_flag=true + break + elif [ "$input" = "n" ]; then + echo -e "Aborting test run..." + echo -e "To run the tests later type: " + echo -e " >>> bash savu_v2.1/savu_installer.sh --tests_only" + exit 0 + else + echo -e "\nYour input was unknown.\n" + fi + done + fi + + setup_script=$PREFIX/'savu_setup.sh' + echo -e "\nCreating a Savu setup script" $setup_script + ( [ -e "$setup_script" ] || touch "$setup_script" ) && [ ! -w "$setup_script" ] && echo cannot write to $setup_script && exit 1 + MPIHOME="$(dirname "$(dirname $MPICC)")" + echo '#!bin/bash' > $setup_script + echo ""export PATH=$MPIHOME/bin:'$PATH'"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$MPIHOME/lib:'$LD_LIBRARY_PATH'"" >> $setup_script + echo ""export PYTHONUSERSITE True"" >> $setup_script + echo ""export PATH=$PYTHONHOME:'$PATH'"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$PYTHONHOME/lib:'$LD_LIBRARY_PATH'"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$astra_lib_path:'$LD_LIBRARY_PATH'"" >> $setup_script + if [ "$CUDAHOME" ]; then + echo ""export PATH=$CUDAHOME/bin:'$PATH'"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$CUDAHOME/lib64:'$LD_LIBRARY_PATH'"" >> $setup_script + fi + if [ "$FFTWHOME" ]; then + echo ""export FFTWDIR=$FFTWHOME"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$FFTWHOME/lib:'$LD_LIBRARY_PATH'"" >> $setup_script + fi + + source $setup_script +fi + +if [ $test_flag ] ; then + + nGPUs=$(nvidia-smi -L | wc -l) + + echo -e "\n***** Testing Savu setup *****\n" + savu_quick_tests + + echo -e "\n*****Running Savu single-threaded local tests *****\n" + savu_full_tests + + echo -e "\n************** Single-threaded local tests complete ******************\n" + + test_dir=`mktemp -d` + tmp_dir=`mktemp -d` + tmpfile=$tmp_dir/temp_output.txt + touch $tmpfile + +fi + +if [ ! $test_flag ] ; then + + launcher_path=`command -v savu_launcher.sh` + mpijob_path=`command -v savu_mpijob.sh` + echo -e "\n\n===============================IMPORTANT NOTICES================================" + echo -e "If you are installing Savu for cluster use, you will need to update the savu " + echo -e "launcher scripts:" + echo -e "\n$launcher_path" + echo -e "$mpijob_path\n" + echo -e "\n\nTo run Savu type 'source $savu_setup' to set relevant paths every time you" + echo -e "open a new terminal. Alternatively, if you are using the Modules system, see" + echo -e "$DIR/module_template for an example module file." + echo -e "================================================================================\n" + + echo -e "*************** SAVU INSTALLATION COMPLETE! ******************\n" + echo -e " ......Thank you for running the Savu installer......\n" + echo -e "=============================================================\n" +else + echo -e "\n\n*************** SAVU TESTS COMPLETE! ******************\n" + echo -e " ......Thank you for running the Savu tests......\n" + echo -e " Please check $tmpfile for errors\n" + echo -e "=======================================================\n" +fi + +exit 0 + diff --git a/install/2_3_1_install/savu_v2.3.1/savu_master/build.sh b/install/2_3_1_install/savu_v2.3.1/savu_master/build.sh new file mode 100644 index 000000000..b6fbaff2b --- /dev/null +++ b/install/2_3_1_install/savu_v2.3.1/savu_master/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +$PYTHON setup.py install --facility $FACILITY # Python command to install the script. + diff --git a/install/2_3_1_install/savu_v2.3.1/savu_master/meta.yaml b/install/2_3_1_install/savu_v2.3.1/savu_master/meta.yaml new file mode 100644 index 000000000..63c862f93 --- /dev/null +++ b/install/2_3_1_install/savu_v2.3.1/savu_master/meta.yaml @@ -0,0 +1,24 @@ +{% set repo = "master" %} + +package: + name: savu + version: "master" + +source: + url: https://github.com/DiamondLightSource/Savu/archive/{{ repo }}.tar.gz + +requirements: + build: + - python + +build: + preserve_egg_dir: True + number: 0 + script_env: + - FACILITY + +about: + home: https://github.com/diamondlightsource/Savu + license: Apache V2 and GPL V3 + license_file: LICENCE.txt + diff --git a/install/2_3_1_install/savu_v2.3.1/version.txt b/install/2_3_1_install/savu_v2.3.1/version.txt new file mode 100644 index 000000000..090c0b654 --- /dev/null +++ b/install/2_3_1_install/savu_v2.3.1/version.txt @@ -0,0 +1,2 @@ +2.3.1 + diff --git a/install/2_3_1_install/savu_v2.3.1/versions_file.txt b/install/2_3_1_install/savu_v2.3.1/versions_file.txt new file mode 100644 index 000000000..116a12c1a --- /dev/null +++ b/install/2_3_1_install/savu_v2.3.1/versions_file.txt @@ -0,0 +1,7 @@ +miniconda 4.5.4 # change this to 'latest' for latest version +mpi4py 3.0.0 +tomopy 1.0.1 +dxchange 0.1.1 +ccpi 0.10.0 +pyfai 0.15.0 + diff --git a/install/2_3_1_install/tests/local_mpi_cpu_test.sh b/install/2_3_1_install/tests/local_mpi_cpu_test.sh new file mode 100644 index 000000000..586af6a0b --- /dev/null +++ b/install/2_3_1_install/tests/local_mpi_cpu_test.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# function for parsing optional arguments +function arg_parse () +{ + flag=$1 + return=$2 + while [[ $# -gt 1 ]] ; do + if [ $1 == $flag ] ; then + eval "$return"=$2 + fi + shift + done +} + +output=$1 +if ! [ "$output" ]; then + echo "Please pass the path to the output folder." + exit 1 +fi + +# set the TESTDATA environment variable +. test_setup.sh + +echo "Running the mpi cpu test..." + + +arg_parse "-r" redirect "$@" +if [ $redirect ] ; then + exec 4<&2 + exec 5<&1 + exec &> $redirect +fi + +savu_mpijob_local.sh $TESTDATA/data/24737.nxs $TESTDATA/test_process_lists/mpi_cpu_test.nxs $output + +if [ $redirect ] ; then + exec 2<&4 + exec 1<&5 +fi + diff --git a/install/2_3_1_install/tests/local_mpi_gpu_test.sh b/install/2_3_1_install/tests/local_mpi_gpu_test.sh new file mode 100644 index 000000000..44cf9bbfd --- /dev/null +++ b/install/2_3_1_install/tests/local_mpi_gpu_test.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# function for parsing optional arguments +function arg_parse () +{ + flag=$1 + return=$2 + while [[ $# -gt 1 ]] ; do + if [ $1 == $flag ] ; then + eval "$return"=$2 + fi + shift + done +} + +output=$1 +if ! [ "$output" ]; then + echo "Please pass the path to the output folder." + exit 1 +fi + +# set the TESTDATA environment variable +. test_setup.sh + +echo "Running the mpi gpu test..." + + +arg_parse "-r" redirect "$@" +if [ $redirect ] ; then + exec 4<&2 + exec 5<&1 + exec &> $redirect +fi + +savu_mpijob_local.sh $TESTDATA/data/24737.nxs $TESTDATA/test_process_lists/mpi_gpu_test.nxs $output + +if [ $redirect ] ; then + exec 2<&4 + exec 1<&5 +fi + diff --git a/install/2_3_1_install/tests/mpi_cpu_test.sh b/install/2_3_1_install/tests/mpi_cpu_test.sh new file mode 100644 index 000000000..105afebcd --- /dev/null +++ b/install/2_3_1_install/tests/mpi_cpu_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +output=$1 +if ! [ "$output" ]; then + echo "Please pass the path to the output folder." + exit 1 +fi + +# get the loaded module version of savu +IN=$LOADEDMODULES +while IFS=':' read -ra ADDR; do + for i in "${ADDR[@]}"; do + if [[ $i == *"savu"* ]]; then + version=`echo "$i" | awk -F/ '{print $NF}'` + fi + done +done <<< "$IN" + +# set the TESTDATA environment variable +. test_setup.sh + +echo "Running the mpi cpu test..." +savu_launcher.sh $version $TESTDATA/data/24737.nxs $TESTDATA/test_process_lists/mpi_cpu_test.nxs $output + diff --git a/install/2_3_1_install/tests/mpi_gpu_test.sh b/install/2_3_1_install/tests/mpi_gpu_test.sh new file mode 100644 index 000000000..fa07b4653 --- /dev/null +++ b/install/2_3_1_install/tests/mpi_gpu_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +output=$1 +if ! [ "$output" ]; then + echo "Please pass the path to the output folder." + exit 1 +fi + +# get the loaded module version of savu +IN=$LOADEDMODULES +while IFS=':' read -ra ADDR; do + for i in "${ADDR[@]}"; do + if [[ $i == *"savu"* ]]; then + version=`echo "$i" | awk -F/ '{print $NF}'` + fi + done +done <<< "$IN" + +# set the TESTDATA environment variable +. test_setup.sh + +echo "Running the mpi gpu test..." +savu_launcher.sh $version $TESTDATA/data/24737.nxs $TESTDATA/test_process_lists/mpi_gpu_test.nxs $output + diff --git a/install/2_3_1_install/tests/test_setup.sh b/install/2_3_1_install/tests/test_setup.sh new file mode 100644 index 000000000..90a35b203 --- /dev/null +++ b/install/2_3_1_install/tests/test_setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +savu_path=$(python -c "import savu; import os; print os.path.abspath(os.path.dirname(savu.__file__))") +test_path=$savu_path'/../test_data' +export TESTDATA=$test_path +echo '$TESTDATA points to the Savu test data folder at' $TESTDATA + diff --git a/install/all_versions.txt b/install/all_versions.txt index 614cf5a4d..1d6d7caec 100644 --- a/install/all_versions.txt +++ b/install/all_versions.txt @@ -4,5 +4,5 @@ 2.2 https://github.com/DiamondLightSource/Savu/raw/v2.2/install/2_1_install/savu_v2.2.tar.gz 2.2.1 https://github.com/DiamondLightSource/Savu/raw/v2.2.1/install/2_1_install/savu_v2.2.1.tar.gz 2.3 https://github.com/DiamondLightSource/Savu/raw/v2.3/install/2_3_install/savu_v2.3.tar.gz -2.3.1 https://github.com/DiamondLightSource/Savu/raw/v2.3.1/install/2_3_install/savu_v2.3.1.tar.gz +2.3.1 https://github.com/DiamondLightSource/Savu/raw/v2.3.1/install/2_3_1_install/savu_v2.3.1.tar.gz diff --git a/install/latest_version.txt b/install/latest_version.txt index 4de92dc40..ddbbf11fa 100644 --- a/install/latest_version.txt +++ b/install/latest_version.txt @@ -1,2 +1,2 @@ -2_3_install/savu_v2.3.1/version.txt +2_3_1_install/savu_v2.3.1/version.txt