Merge pull request #799 from DiamondLightSource/nics_tools

citation doc

doc/source/howto/install/savu_hpc.rst

@@ -10,7 +10,7 @@ performance, we recommend a fast network interconnect, such as infiniBand, and
 a high performance parallel filesystem, such as GPFS or Lustre.
 
 Requirements:
-    - A Unix system with an `openMPI <https://www.open-mpi.org/>`_ library installed.
+    - A Unix system with MPI libraries installed: Savu has been test with `openMPI <https://www.open-mpi.org/>`_ (latest version 4.1.1)
 
 
 **Installation of Savu HPC outside Diamond Light Source (DLS) systems**

doc/source/index.rst

@@ -66,6 +66,15 @@ Table of Contents
 
     reference/plugin_autosummary
 
+.. _citations_content:
+
+.. toctree::
+	:caption: Citations
+	:maxdepth: 2
+
+	reference/savu_citations
+
+
 Indices and tables
 ==================
 * :ref:`genindex`

doc/source/reference/savu_citations.rst

@@ -0,0 +1,23 @@
+
+How to cite Savu software
+=========================
+
+.. start_of_main_text
+
+The relevant citations for your Savu job are contained in the "citations.txt" file in the Savu output folder, in BibTeX and EndNote formats.  These include:
+
+* Savu framework citations
+* Plugin citations - specific to your process list
+
+All citations are also contained in both the Savu process list and output Savu nexus file and can be extracted into BibTeX and EndNote formats using the savu_citations tool.  From the command line run:
+
+>>> savu_citations <your_nxs_file> <out_file_name>
+
+Where the output file is the name of the file you would like to extract this information to.
+
+For example,
+
+>>> savu_citations  process_lists.nxs  citations.txt
+
+*** Please include all Savu citations in your relevant publications ***
+

install/installation_notes.txt

@@ -0,0 +1,22 @@
+Installation notes (for creating a new release and installing at Diamond)
+=========================================================================
+
+- Freeze on pull requests
+- Install latest version of Savu from Master into /dls_sw/apps/savu
+- Create savu_plugins folder in /dls_sw/apps/savu/$release/
+- Create a new module file
+- Test new installation
+- Update Savu/install/latest_version.py to contain the new version number (and the corresponding file)
+- Update installation notes with the latest tested versions of libraries
+- Create tar.gz of the new install folder
+   >>> tar -zcvf savu_installer.tar.gz Savu/install/savu_hpc/savu_installer
+- Push all changes to Github (ensure all tests passing) and check download link works correctly
+- Collate release notes
+	Get all Git commits since the last release: git log --oneline v3.0.. HEAD > a.txt
+	Get all Git files added since the last release: git diff v3.0 HEAD --name-status | grep A
+- Create release on Github
+- Reinstall Savu into new release Conda environment
+- Further testing
+- (Make this module default load if during shutdown)
+- Email beamlines
+

savu/tomo_recon.py

@@ -30,6 +30,7 @@
 from savu.version import __version__
 
 import savu.core.utils as cu
+from scripts.citation_extractor import citation_extractor
 from savu.core.basic_plugin_runner import BasicPluginRunner
 from savu.core.plugin_runner import PluginRunner
 
@@ -217,6 +218,9 @@ def main(input_args=None):
     try:
         plugin_runner = pRunner(options)
         plugin_runner._run_plugin_list()
+        if options['process'] == 0:
+            in_file = plugin_runner.exp.meta_data['nxs_filename']
+            citation_extractor.main(in_file=in_file, quiet=True)
     except Exception:
         # raise the error in the user log
         trace = traceback.format_exc()

scripts/citation_extractor/citation_extractor.py

@@ -8,8 +8,8 @@
 
 class NXcitation(object):
     def __init__(self, description, doi, endnote, bibtex):
-        self.description = description
-        self.doi = doi
+        self.description = description.decode('UTF-8')
+        self.doi = doi.decode('UTF-8')
         self.endnote = endnote.decode('UTF-8')
         self.bibtex = bibtex.decode('UTF-8')
 
@@ -104,14 +104,23 @@ def __option_parser(doc=True):
     return parser if doc==True else parser.parse_args()
 
 
-def main():
-    args = __option_parser(doc=False)
-    infile = h5py.File(args.in_file, 'r')
+def main(in_file=None, quiet=False):
+    # when calling directly from tomo_recon.py
+    if in_file:
+        out_file = os.path.join(os.path.dirname(in_file), 'citations.txt')
+    else:
+        args = __option_parser(doc=False)
+        in_file = args.in_file
+        out_file = args.out_file
+
+    infile = h5py.File(in_file, 'r')
     citation_manager = NXciteVisitor().get_citation_manager(infile, '/')
     if citation_manager is not None:
-        with open(args.out_file, 'w') as outfile:
+        with open(out_file, 'w') as outfile:
             outfile.write(citation_manager.__str__())
-    print("Extraction complete")
+
+    if not quiet:
+        print("Extraction complete")
 
 if __name__ == '__main__':
     main()

system_files/dls/modulefile/3.0 → system_files/dls/modulefile/4.0

@@ -13,46 +13,29 @@ proc ModulesHelp { } {
 
 module load global/directories
 
+
 if { [info exists env(SAVUHOME)] } {
 	set gitrepo $env(SAVUHOME)
 }
 
-# Set RHEL 6 or 7
-# set redhatrelease [eval exec "cat /etc/redhat-release"]
-# if { [regexp -nocase {release 6} $redhatrelease] } {
-# 	set OS rhel6
-# } elseif { [regexp -nocase {release 7} $redhatrelease] } {
-# 	set OS rhel7
-# } else {
-# 	puts stderr "Incompatible operating system."
-# }
-
-# # switch OS
-# if { [info exists env(SWITCH_SAVU_OS)] } {
-# 	if { [string equal $OS rhel7] } {
-# 		set OS rhel6
-# 	} else {
-# 		set OS rhel7
-# 	}
-# }
-
-#============================================================================
-# When creating a new module, update these variables.
-set savu_version 3.0
-
-set mpi_version 3.1.4
-# if { [string equal $OS rhel6] } {
-# 	set PYTHON_HOME $env(softwaredir)/savu/2.4_RHEL6/miniconda
-#     setenv SAVU_PLUGINS_PATH /dls_sw/apps/savu/2.4_RHEL6/savu_plugins
-# 	#puts stderr "\tLoading Savu for RHEL 6"
-# } else {
-	set PYTHON_HOME $env(softwaredir)/savu/3.0_RHEL7/savu
-	# set PYTHON_HOME $env(softwaredir)/savu/3.0_RHEL7/miniconda
-    setenv SAVU_PLUGINS_PATH /dls_sw/apps/savu/3.0_RHEL7/savu_plugins
-	#puts stderr "\tLoading Savu for RHEL 7"
-# }
-
-#============================================================================
+
+set hostname [eval exec "cat /proc/sys/kernel/hostname"]
+if { [regexp -nocase {cs03r-sc-serv-39} $hostname] } {
+	puts stderr "\t Unfortunately Savu won't work on this node, please start a new nx session."
+	exit
+} elseif { [regexp -nocase {cs03r-sc-serv-40} $hostname] } {
+	puts stderr "\t Unfortunately Savu won't work on this node, please start a new nx session."
+	exit
+} else {
+	#============================================================================
+	# When creating a new module, update these variables.
+	set savu_version 4.0
+	set mpi_version 4.1.1
+	set PYTHON_HOME /dls_sw/apps/savu/4.0/miniconda
+	set hebi /dls_sw/apps/savu/hebi
+	#============================================================================	
+}
+
 
 set mach $tcl_platform(machine)
 module-whatis "loads Savu version $savu_version"
@@ -74,6 +57,7 @@ if { [module-info mode load] } {
     module load openmpi/$mpi_version
   }
 
+
   if { ! [info exists env(SAVU_PRERELEASE)] } {
   	puts stderr "\n\t\t    ********************************************"
   	puts stderr "\t\t    *         Loading Savu version $savu_version         *"
@@ -91,37 +75,35 @@ if { [module-info mode load] } {
     module unload openmpi/$mpi_version
   }
 
-  # TODO is this necessary? SAVUHOME seems like a user-set environment
-  # so we shouldn't automatically delete it
-  # if { [info exists env(SAVUHOME)] } {
-	# unsetenv SAVUHOME
-	# puts stderr "Cleaning up SAVUHOME"
-  # }
-
-  # if { [info exists env(SWITCH_SAVU_OS)] } {
-	# unsetenv SWITCH_SAVU_OS
-	# puts stderr "Cleaning up SWITCH_SAVU_OS"
-  # }
+  if { [info exists env(SAVUHOME)] } {
+	unsetenv SAVUHOME
+	puts stderr "Cleaning up SAVUHOME"
+  }
+
 }
 
 setenv PYTHONNOUSERSITE True
 prepend-path PATH $PYTHON_HOME/bin
+prepend-path PATH $hebi
 
 module add fastxrf
 
 set-alias "savu_mpi" "savu_launcher.sh -s $savu_version"
 set-alias "savu_mpi_auto" "savu_launcher.sh -s $savu_version -t AUTO"
 set-alias "savu_mpi_preview" "savu_launcher.sh -s $savu_version -t PREVIEW"
+set-alias "savu_mpi_com14" "savu_launcher.sh -s $savu_version -t BIG"
 
 if { [info exists gitrepo] } {
   prepend-path PATH $gitrepo
   prepend-path PYTHONPATH $gitrepo
+  prepend-path PYTHONPATH $gitrepo/scripts  # required for the configurators
   set-alias "savu_mpi_dev" "$gitrepo/system_files/dls/mpi/savu_launcher.sh -s $savu_version -i"
   set-alias "savu_mpi_local" "$gitrepo/system_files/dls/mpi/savu_mpijob_local.sh"
-  set-alias "savu_config" "python $gitrepo/scripts/config_generator/savu_config.py"
+  set-alias "savu_config" "python -m config_generator.savu_config"
   set-alias "savu" "python $gitrepo/savu/tomo_recon.py"
   set-alias "savu_citations" "python $gitrepo/scripts/citation_extractor/citation_extractor.py"
 } else {
   set-alias "savu" "savu -s cs04r-sc-serv-14"
   set-alias "savu_mpi_local" "savu_mpijob_local.sh"
 }
+