cleanup-cmssdt

#!/bin/sh -ex

DRY_RUN=$1
if [ "X$DRY_RUN" = "Xtrue" ] ; then
  DRY_RUN=echo
else
  DRY_RUN=
fi

JENKINS_ARTIFACTS=/data/sdt/SDT/jenkins-artifacts
CVMFS_JENKINS_ARTIFACTS=/cvmfs/cms-ib.cern.ch/jenkins-artifacts
DIRS_PROCESSED=
JOBS_TO_KEEP=300
RUNS_TO_KEEP=3
#Keep $JOBS_TO_KEEP of these
for dir in pull-request-integration ; do
  [ -d ${JENKINS_ARTIFACTS}/$dir ] || continue
  DIRS_PROCESSED="${DIRS_PROCESSED} ${dir}"
  cd ${JENKINS_ARTIFACTS}/$dir
  find . -maxdepth 3 -mindepth 3 -name '.keep' -type f | sed 's|^./||;s|/.*||' | xargs -i --no-run-if-empty touch "./{}"
  ls -rt | head -n -$JOBS_TO_KEEP | xargs -i --no-run-if-empty $DRY_RUN rm -rf '{}'
  for run in `ls -d *`; do
    ls -d $run/* | sed 's|.*/||' | sort -n | head -n -$RUNS_TO_KEEP | xargs -i --no-run-if-empty $DRY_RUN rm -rf "$run/{}"
  done
done

RUNS_TO_KEEP=1
#Keep $JOBS_TO_KEEP of these with RUNS_TO_KEEP
for dir in  baseLineComparisons baseLineComparisonsGPU ; do
  [ -d ${JENKINS_ARTIFACTS}/$dir ] || continue
  DIRS_PROCESSED="${DIRS_PROCESSED} ${dir}"
  cd ${JENKINS_ARTIFACTS}/$dir
  ls -rt | head -n -$JOBS_TO_KEEP | xargs -i --no-run-if-empty $DRY_RUN rm -rf '{}'
  for run in `ls -d *`; do
    ls -d $run/* | sed 's|.*/||' | sort -n | head -n -$RUNS_TO_KEEP | xargs -i --no-run-if-empty $DRY_RUN rm -rf "$run/{}"
  done
done

JOBS_TO_KEEP=50
for dir in material-budget-ref ; do
  [ -d ${JENKINS_ARTIFACTS}/$dir ] || continue
  DIRS_PROCESSED="${DIRS_PROCESSED} ${dir}"
  cd ${JENKINS_ARTIFACTS}/$dir
  ls -rt | head -n -$JOBS_TO_KEEP | xargs -i --no-run-if-empty $DRY_RUN rm -rf '{}'
done

#Keep the following for 15 days
DAYS_TO_KEEP=15
DIRS="lizard flawfinder invalid-includes"
DIRS="${DIRS} check_headers valgrind HLT-Validation ib-static-analysis ib-baseline-tests ib-dqm-tests igprof"
DIRS="${DIRS} iwyu material-budget das_query build-any-ib"
for dir in ${DIRS}; do
  [ -d ${JENKINS_ARTIFACTS}/$dir ] || continue
  DIRS_PROCESSED="${DIRS_PROCESSED} ${dir}"
  find ${JENKINS_ARTIFACTS}/${dir} -mtime +${DAYS_TO_KEEP} -mindepth 1 -maxdepth 1 | xargs -i --no-run-if-empty $DRY_RUN rm -rf '{}'
done

#Keep the following for 90 days
DAYS_TO_KEEP=30
DIRS="run-pr-format pr-code-checks jenkins-test-code-format profiling"
DIRS="${DIRS} auto-build-release auto-upload-release cleanup-auto-build build-fwlite release-run-reco-profiling"
for dir in ${DIRS}; do
  [ -d ${JENKINS_ARTIFACTS}/$dir ] || continue
  DIRS_PROCESSED="${DIRS_PROCESSED} ${dir}"
  find ${JENKINS_ARTIFACTS}/${dir} -mtime +${DAYS_TO_KEEP} -mindepth 1 -maxdepth 1 | xargs -i --no-run-if-empty $DRY_RUN rm -rf '{}'
done

#Cleanup dirs which are available on CVMFS
for dir in ib-baseline-tests ; do
  for ver in $(find ${JENKINS_ARTIFACTS}/${dir}  -maxdepth 2 -mindepth 2 -path "*/${dir}/CMSSW_*/*" -type d | sed "s|${JENKINS_ARTIFACTS}/||") ; do
    if [ -d ${CVMFS_JENKINS_ARTIFACTS}/${ver} ] ; then
      $DRY_RUN rm -rf ${JENKINS_ARTIFACTS}/${ver}
      $DRY_RUN ln -s ${CVMFS_JENKINS_ARTIFACTS}/${ver} ${JENKINS_ARTIFACTS}/${ver}
    fi
  done
  for l in $(ls -d ${JENKINS_ARTIFACTS}/${dir}/CMSSW_*) ; do 
    if  [ -L $l -a ! -e $l ] ; then
      rm -f $l
    fi
  done
done

NEW_DIRS=""
for dir in `find ${JENKINS_ARTIFACTS} -mindepth 1 -maxdepth 1 -type d | sed 's|.*/||'`; do
  if [ "X`echo ${DIRS_PROCESSED} | grep ${dir}`" = "X" ] ; then
    NEW_DIRS="${NEW_DIRS} ${dir}"
  fi
done

if [ "X${NEW_DIRS}" != "X" ] ; then
  echo -e "${BUILD_URL}\nNew directories: ${NEW_DIRS}" | mail -s '[CMSSDT] Found new Directories to cleanup' cms-sdt-logs@cern.ch
fi

WEB_LOG_DIR=/data/sdt/buildlogs
find /data/sdt/buildlogs -mindepth 3 -maxdepth 3 -mtime +15  -path '*/fwlite/CMSSW_*' -type d | xargs -i --no-run-if-empty $DRY_RUN rm -rf '{}'
find /data/sdt/buildlogs -mindepth 5 -maxdepth 5 -mtime +15  -path '*/www/*/CMSSW_*' -type d | xargs -i --no-run-if-empty $DRY_RUN rm -rf '{}'