Keep calling rsync after aws download to get latest changes

sokrypton · Feb 4, 2024 · ef46c9d · ef46c9d
1 parent e8e3610
commit ef46c9d
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 9 deletions.
diff --git a/MsaServer/setup-and-start-local.sh b/MsaServer/setup-and-start-local.sh
@@ -24,7 +24,7 @@ MMSEQS_COMMIT=${1:-4589151554eb83a70ff0c4d04d21b83cabc203e4}
 BACKEND_COMMIT=${2:-14e087560f309f989a5e1feb54fd1f9c988076d5}
 
 # check if all dependencies are there
-for i in go curl git aria2c rsync aws; do
+for i in go curl git aria2c rsync; do
   if ! command -v "${i}" > /dev/null 2>&1; then
     echo "${i} is not installed, please install it first"
     exit 1

diff --git a/setup_databases.sh b/setup_databases.sh
@@ -6,9 +6,13 @@ WORKDIR="${1:-$(pwd)}"
 
 PDB_SERVER="${2:-"rsync.wwpdb.org::ftp"}"
 PDB_PORT="${3:-"33444"}"
+# do initial download of the PDB through aws?
+# still syncs latest structures through rsync
+PDB_AWS_DOWNLOAD="${4:-}"
+PDB_AWS_SNAPSHOT="20240101"
+
 UNIREF30DB="uniref30_2302"
 MMSEQS_NO_INDEX=${MMSEQS_NO_INDEX:-}
-AWS_MMCIF_DOWNLOAD="false"
 cd "${WORKDIR}"
 
 hasCommand () {
@@ -23,6 +27,12 @@ if [ "$STRATEGY" = "" ]; then
 	    fail "No download tool found in PATH. Please install aria2c, curl or wget."
 fi
 
+if [ -n "${PDB_AWS_DOWNLOAD}" ]; then
+    if ! hasCommand aws; then
+        fail "aws command not found in PATH. Please install the aws command line tool."
+    fi
+fi
+
 downloadFile() {
     URL="$1"
     OUTPUT="$2"
@@ -95,12 +105,11 @@ fi
 if [ ! -f PDB_MMCIF_READY ]; then
   mkdir -p pdb/divided
   mkdir -p pdb/obsolete
-  if [ "$AWS_MMCIF_DOWNLOAD" = "true" ]; then
-    aws s3 cp --no-sign-request s3://pdbsnapshots/20240101/pub/pdb/data/structures/divided/mmCIF/ pdb/divided/ --recursive
-    aws s3 cp --no-sign-request s3://pdbsnapshots/20240101/pub/pdb/data/structures/obsolete/mmCIF/ pdb/obsolete/ --recursive
-  else
-    rsync -rlpt -v -z --delete --port=${PDB_PORT} ${PDB_SERVER}/data/structures/divided/mmCIF/ pdb/divided
-    rsync -rlpt -v -z --delete --port=${PDB_PORT} ${PDB_SERVER}/data/structures/obsolete/mmCIF/ pdb/obsolete
+  if [ -n "${PDB_AWS_DOWNLOAD}" ]; then
+    aws s3 cp --no-sign-request --recursive s3://pdbsnapshots/${PDB_AWS_SNAPSHOT}/pub/pdb/data/structures/divided/mmCIF/ pdb/divided/
+    aws s3 cp --no-sign-request --recursive s3://pdbsnapshots/${PDB_AWS_SNAPSHOT}/pub/pdb/data/structures/obsolete/mmCIF/ pdb/obsolete/
   fi
+  rsync -rlpt -v -z --delete --port=${PDB_PORT} ${PDB_SERVER}/data/structures/divided/mmCIF/ pdb/divided
+  rsync -rlpt -v -z --delete --port=${PDB_PORT} ${PDB_SERVER}/data/structures/obsolete/mmCIF/ pdb/obsolete
   touch PDB_MMCIF_READY
-fi
+fi