Merge pull request #75 from DiamondLightSource/prep_new_v1.20.0

Prepare new version 1.20.0

HISTORY.rst

@@ -5,6 +5,44 @@ History
 Unreleased / master
 -------------------
 
+1.20.0 (2021-04-13)
+-------------------
+
+* Tables, views and indices:
+
+ * New cryoEM table `ParticleClassificationGroup`. Some columns were moved to this table from `ParticlePicker` and `ParticleClassification`.
+ * Add `capillary` as enum option to `BLSampleGroup_has_BLSample.type`
+ * XChem DB: First version of schema for production (not part of release)
+ * Analytics schema: Add access to all lookup tables, `RobotAction`, `ProcessingJob*`, `Protein` tables through views (not part of release)
+ * Drop duplicate indices, add missing indices that existed only in DLS prod
+
+* Data in lookup tables:
+
+ * Add additional container types: some historical, some new
+ * Update `ExperimentType.proposalType`: Change scm to saxs, MX to mx
+ * Update `ContainerType.proposalType`: Change scm to saxs
+ * Add two XPDF and five MX container types
+ * Update `SpaceGroup` table: Remove newlines in names. Set `MX_used` = 1 for all SGs.
+
+* Stored procedures:
+
+ * `retrieve_scm_*``: Preferentially get experiement type and container type+capacity from new lookup tables `ExperimentType` and `ContainerType`, otherwise fall-back to `experimentType` and `containerType`+`capacity` columns in the `Container` table.
+ * `upsert_sample_image`: set `BLSampleImage.modifiedTimeStamp` if the upsert is an update
+
+* Tools and documentation:
+
+ * New script `bin/missed_updates.sh` to identify update .sql files that haven't been run
+ * `bin/backup.sh`: Use `--add-drop-trigger` flag to drop trigger if exists before creation
+ * `CONTRIBUTING.md`: Simplify pull request procedure for table changes
+ * Updated simplified MX database diagram files
+ * Move list + string with lookup tables to separate file so it can be reused
+ * All bash scripts have been given new shebang lines to run on systems where bash is not at `/bin/bash`.
+
+* Users, roles and grants:
+
+ * More `INSERT` grants for `ispyb_web` role: DC, DCG, `ProcessingJob`, `ProcessingJobParameter`
+ * New `ispyb_ro_nopii` role (read-only-except-PII)
+
 1.19.0 (2021-03-05)
 -------------------
 

bin/add_schema_status_stmts.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 fname=$(basename ${1})
 

bin/backup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Usage example:
 # backup.sh ../dbbackups

bin/db_procs_to_rst.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 db=$1
 

bin/db_tables_to_rst.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 db=$1
 mysql --defaults-file=../.my.cnf -D $db --skip-column-names --batch --raw < sql/db_tables_to_tsv.sql > /tmp/tmp1.tsv

bin/export_session.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Usage:
 # export_session.sh <.cnf file> <db_name> <proposalCode><proposalNumber> <session number> <output dir>

bin/forget_all_labs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes details about laboratories on a particular proposal
 # and session.

bin/forget_all_persons.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Caution: This script attempts to remove all Personally Identifiable
 # Information (PII) in the given ispyb database for the given proposal

bin/forget_lab.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes details about a particular laboratory.
 # Usage:

bin/forget_pdb.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular PDB.
 #

bin/forget_person.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes Personally Identifiable Information (PII) for a particular person.
 #

bin/forget_protein.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular protein.
 #

bin/forget_sample.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular sample (BLSample).
 #

bin/forget_shipping.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular shipping.
 #

bin/forget_subsample.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular sub-sample (BLSubSample).
 #

bin/forget_unused_shipments.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes all descriptive information related to shipments for
 # a particular proposal, except that related to samples on which data was

bin/lookup_tables.sh

@@ -4,6 +4,8 @@ SchemaStatus
 ComponentType
 ComponentSubType
 ConcentrationType
+ContainerType
+ExperimentType
 InspectionType
 PlateType
 SessionType

bin/mdb_cluster_backup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash 
+#!/usr/bin/env bash
 
 # This script assumes a MariaDB Galera Cluster with read-write split. The
 # script must be run locally on one of the database read nodes.

bin/missed_updates.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Get the database name
+if [ -z "${DB}" ]
+then
+ DB="ispyb_build"
+fi
+
+# Get this scripts dir
+dir=$(dirname $(realpath -e ${0}))
+
+# Get the project'r root dir
+project_root=$(dirname "${dir}")
+
+# Load some function definitions in case we need them
+source ${project_root}/bin/functions.sh
+
+# Get a list of all update files
+all_sql_files=`cd ${project_root}/schemas/ispyb/updates && ls *.sql && cd ../..`
+
+# Get a list of all update files recorded in the SchemaStatus table
+done_sql_files=`mysql --defaults-file=${project_root}/.my.cnf -D $DB --skip-column-names --silent --raw -e "SELECT scriptName FROM SchemaStatus WHERE schemaStatus = 'DONE' ORDER BY recordTimeStamp;"`
+
+# Make a list of all update files not recorded in the SchemaStatus table
+arr=()
+while read -r sql_file; do
+ contains "$done_sql_files" "$sql_file"
+ if [[ $? -ne 0 ]]
+ then
+ echo "$sql_file"
+ fi
+done <<< "$all_sql_files"

build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Script to create the database from the schema .sql files and identify any
 # update .sql files that haven't been run. Also generate documentation from
@@ -11,44 +11,32 @@
 
 source bin/functions.sh
 
-if [ -z "${db}" ]
+if [ -z "${DB}" ]
 then
- db="ispyb_build"
+ DB="ispyb_build"
 fi
 
 echo "Dropping + creating build database"
-mysql --defaults-file=.my.cnf -e "DROP DATABASE IF EXISTS $db; CREATE DATABASE $db; SET GLOBAL log_bin_trust_function_creators=ON;"
+mysql --defaults-file=.my.cnf -e "DROP DATABASE IF EXISTS $DB; CREATE DATABASE $DB; SET GLOBAL log_bin_trust_function_creators=ON;"
 
 if [[ $? -eq 0 ]]
 then
- mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/tables.sql
- mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/lookups.sql
- mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/data.sql
- mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/routines.sql
- mysql --defaults-file=.my.cnf -D $db < grants/ispyb_acquisition.sql
- mysql --defaults-file=.my.cnf -D $db < grants/ispyb_processing.sql
- mysql --defaults-file=.my.cnf -D $db < grants/ispyb_web.sql
- mysql --defaults-file=.my.cnf -D $db < grants/ispyb_import.sql
+ mysql --defaults-file=.my.cnf -D $DB < schemas/ispyb/tables.sql
+ mysql --defaults-file=.my.cnf -D $DB < schemas/ispyb/lookups.sql
+ mysql --defaults-file=.my.cnf -D $DB < schemas/ispyb/data.sql
+ mysql --defaults-file=.my.cnf -D $DB < schemas/ispyb/routines.sql
+ mysql --defaults-file=.my.cnf -D $DB < grants/ispyb_acquisition.sql
+ mysql --defaults-file=.my.cnf -D $DB < grants/ispyb_processing.sql
+ mysql --defaults-file=.my.cnf -D $DB < grants/ispyb_web.sql
+ mysql --defaults-file=.my.cnf -D $DB < grants/ispyb_import.sql
 
- # Identify update .sql files that haven't been run, and run them
- all_sql_files=`cd schemas/ispyb/updates && ls *.sql && cd ../..`
-
- done_sql_files=`mysql --defaults-file=.my.cnf -D $db --skip-column-names --silent --raw -e "SELECT scriptName FROM SchemaStatus WHERE schemaStatus = 'DONE' ORDER BY recordTimeStamp;"`
-
- arr=()
- while read -r sql_file; do
- contains "$done_sql_files" "$sql_file"
- if [[ $? -ne 0 ]]
- then
- arr+=("$sql_file")
- mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/updates/$sql_file
- fi
- done <<< "$all_sql_files"
+ arr=$(bin/missed_updates.sh)
 
  if [ -n "$arr" ]; then
- echo "The following schemas/ispyb/updates/*.sql files were sourced:"
- for val in "${arr[@]}"; do
- echo "$val"
+ echo "Running schemas/ispyb/updates/*.sql files that haven't yet been run:"
+ for sql_file in ${arr[@]}; do
+ echo "$sql_file"
+ mysql --defaults-file=.my.cnf -D ${DB} < "schemas/ispyb/updates/${sql_file}"
  done
  else
  echo "No new schemas/ispyb/updates/*.sql files."
@@ -57,8 +45,8 @@ then
  # Generate table and sproc documentation
  if [ -d "bin" ] && [ -d "docs" ]; then
  cd bin
- ./db_procs_to_rst.sh $db > ../docs/list_of_procs.rst
- ./db_tables_to_rst.sh $db > ../docs/list_of_tables_and_columns.rst
+ ./db_procs_to_rst.sh $DB > ../docs/list_of_procs.rst
+ ./db_tables_to_rst.sh $DB > ../docs/list_of_tables_and_columns.rst
  cd ..
  fi
 

docs/list_of_tables_and_columns.rst

@@ -1317,26 +1317,29 @@
  ``y``,float,""
  **ParticleClassification**,table,"Results of 2D or 2D classification"
  ``particleClassificationId``,int,""
- ``particlePickerId``,int,""
- ``type``,enum,"Indicates the type of particle classification"
- ``batchNumber``,int,"Corresponding to batch number"
  ``classNumber``,int,"Identified of the class. A unique ID given by Relion"
  ``classImageFullPath``,varchar,"The PNG of the class"
- ``numberOfParticlesPerBatch``,int,"total number of particles per batch (a large integer)"
- ``numberOfClassesPerBatch``,int,""
  ``particlesPerClass``,int,"Number of particles within the selected class, can then be used together with the total number above to calculate the percentage"
  ``rotationAccuracy``,int,"???"
  ``translationAccuracy``,float,"Unit: Angstroms"
  ``estimatedResolution``,float,"???, Unit: Angstroms"
  ``overallFourierCompleteness``,float,""
+ ``particleClassificationGroupId``,int,""
+ **ParticleClassificationGroup**,table,""
+ ``particleClassificationGroupId``,int,""
+ ``particlePickerId``,int,""
+ ``programId``,int,""
+ ``type``,enum,"Indicates the type of particle classification"
+ ``batchNumber``,int,"Corresponding to batch number"
+ ``numberOfParticlesPerBatch``,int,"total number of particles per batch (a large integer)"
+ ``numberOfClassesPerBatch``,int,""
  ``symmetry``,varchar,""
  **ParticleClassification_has_CryoemInitialModel**,table,""
  ``particleClassificationId``,int,""
  ``cryoemInitialModelId``,int,""
  **ParticlePicker**,table,"An instance of a particle picker program that was run"
  ``particlePickerId``,int,""
- ``particlePickerProgramId``,int,""
- ``particleClassificationProgramId``,int,""
+ ``programId``,int,""
  ``firstMotionCorrectionId``,int,""
  ``particlePickingTemplate``,varchar,"Cryolo model"
  ``particleDiameter``,float,"Unit: nm"

release.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Script to make a release from code currently in the repo if none exists for
 # the current schema version.

schemas/ispyb/data.sql

@@ -422,14 +422,6 @@ INSERT INTO `ContainerRegistry` (`containerRegistryId`, `barcode`, `comments`, `
 /*!40000 ALTER TABLE `ContainerReport` DISABLE KEYS */;
 /*!40000 ALTER TABLE `ContainerReport` ENABLE KEYS */;
 
---
--- Dumping data for table `ContainerType`
---
-
-/*!40000 ALTER TABLE `ContainerType` DISABLE KEYS */;
-INSERT INTO `ContainerType` (`containerTypeId`, `name`, `proposalType`, `active`, `capacity`, `wellPerRow`, `dropPerWellX`, `dropPerWellY`, `dropHeight`, `dropWidth`, `dropOffsetX`, `dropOffsetY`, `wellDrop`) VALUES (1,'B21_8+1','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(2,'B21_96','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(3,'B21_1tube','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(4,'I22_Capillary_Rack_20','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(5,'I22_Grid_100','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(6,'I22_Grid_45','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(7,'P38_Capillary_Rack_27','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(8,'P38_Solids','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(9,'P38_Powder','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(10,'B22_6','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(11,'I11_Capillary_Rack_6','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(12,'Puck','mx',1,16,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(13,'ReferencePlate','mx',1,16,2,1,1,1,1,0,0,-1),(14,'CrystalQuickX','mx',1,192,12,2,1,0.5,1,0,0,-1),(15,'MitegenInSitu','mx',1,192,12,2,1,0.5,1,0,0,-1),(16,'FilmBatch','mx',1,96,12,1,1,1,1,0,0,-1),(17,'MitegenInSitu_3_Drop','mx',1,288,12,3,1,0.5,1,0,0,-1),(18,'Greiner 3 Drop','mx',1,288,12,3,1,0.5,1,0,0,-1),(19,'MRC Maxi','mx',1,48,6,1,1,1,0.5,0,0,-1),(20,'MRC 2 Drop','mx',1,192,12,1,2,1,0.5,0.5,0,-1),(21,'Griener 1536','mx',1,1536,12,4,4,1,1,0,0,-1),(22,'3 Drop Square','mx',1,288,12,2,2,1,1,0,0,3),(23,'SWISSCI 3 Drop','mx',1,288,12,2,2,1,1,0,0,1),(24,'1 drop','mx',1,96,12,1,1,0.5,0.5,0,0,-1),(25,'LCP Glass','mx',1,96,12,1,1,1,1,0,0,-1),(26,'PCRStrip','scm',1,9,9,1,1,1,1,0,0,-1);
-/*!40000 ALTER TABLE `ContainerType` ENABLE KEYS */;
-
 --
 -- Dumping data for table `CourierTermsAccepted`
 --
@@ -591,14 +583,6 @@ INSERT INTO `EnergyScan` (`energyScanId`, `sessionId`, `blSampleId`, `fluorescen
 /*!40000 ALTER TABLE `ExperimentKindDetails` DISABLE KEYS */;
 /*!40000 ALTER TABLE `ExperimentKindDetails` ENABLE KEYS */;
 
---
--- Dumping data for table `ExperimentType`
---
-
-/*!40000 ALTER TABLE `ExperimentType` DISABLE KEYS */;
-INSERT INTO `ExperimentType` (`experimentTypeId`, `name`, `proposalType`, `active`) VALUES (1,'Default',NULL,1),(2,'MXPressE','mx',1),(3,'MXPressO','mx',1),(4,'MXPressE_SAD','mx',1),(5,'MXScore','mx',1),(6,'MXPressM','mx',1),(7,'MAD','mx',1),(8,'SAD','mx',1),(9,'Fixed','mx',1),(10,'Ligand binding','mx',1),(11,'Refinement','mx',1),(12,'OSC','MX',1),(13,'MAD - Inverse Beam','mx',1),(14,'SAD - Inverse Beam','mx',1),(15,'MESH','mx',1),(16,'XFE','mx',1),(17,'Stepped transmission','mx',1),(18,'XChem High Symmetry','mx',1),(19,'XChem Low Symmetry','mx',1),(20,'Commissioning','mx',1),(21,'HPLC','scm',1),(22,'Robot','scm',1),(23,'Rack','scm',1),(24,'Grid','scm',1),(25,'Solids','scm',1),(26,'Powder','scm',1),(27,'Peltier','scm',1),(28,'Spectroscopy','scm',1),(29,'CD Spectroscopy','scm',1),(30,'Microscopy','scm',1),(31,'Imaging','scm',1),(32,'CD Thermal Melt','scm',1),(33,'Fixed Energy At Ambient With Robot','scm',1);
-/*!40000 ALTER TABLE `ExperimentType` ENABLE KEYS */;
-
 --
 -- Dumping data for table `Frame`
 --
@@ -870,6 +854,13 @@ INSERT INTO `PDB` (`pdbId`, `name`, `contents`, `code`) VALUES (6,'ceo2','\r\nda
 /*!40000 ALTER TABLE `ParticleClassification` DISABLE KEYS */;
 /*!40000 ALTER TABLE `ParticleClassification` ENABLE KEYS */;
 
+--
+-- Dumping data for table `ParticleClassificationGroup`
+--
+
+/*!40000 ALTER TABLE `ParticleClassificationGroup` DISABLE KEYS */;
+/*!40000 ALTER TABLE `ParticleClassificationGroup` ENABLE KEYS */;
+
 --
 -- Dumping data for table `ParticleClassification_has_CryoemInitialModel`
 --
@@ -1457,4 +1448,4 @@ INSERT INTO `v_run` (`runId`, `run`, `startDate`, `endDate`) VALUES (1,'2008-01'
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2021-03-05 16:12:02
+-- Dump completed on 2021-04-13 17:47:06