i#2255: Build cronbuilds as official packages (#2256)

Switches to a separate invocation of package.cmake for cronbuilds, to
make them look like release packages.  Cronbuilds on Travis and
Appveyor no longer run the same jobs as a regular build but instead
run one package.cmake job.

If the build number is 0, we omit it from the package name, rather
than appending it.

.travis.yml now takes in VERSION_NUMBER and uses it directly.  There
is no more TAG_SUFFIX: VERSION_NUMBER has to include the build number
if that's desired, and that will be part of the tag as well.  If
VERSION_NUMBER is not set it uses the default as before with a
cronbuild- prefix for the tag.

runsuite_wrapper.pl parses VERSION_NUMBER and if it has a -NNN it
passes that as the build= arg to package.cmake.  It also passes a
version= argument.

Adds WiX to the PATH for building an .msi package.

Makes a package failure turn the build red.

Installs doxygen from direct download instead of choco.

Updates DR to 2976f031 to pull in the doxyutils fix for identifying cygwin doxygen.

No need to install ghostscript or imagemagick or fig2dev anymore after
DRi#3235.  Removes those installations from Travis and Appveyor.

Fixes #2255

.appveyor.yml

@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2017-2018 Google, Inc.  All rights reserved.
+# Copyright (c) 2017-2020 Google, Inc.  All rights reserved.
 # **********************************************************
 
 # Dr. Memory: the memory debugger
@@ -64,18 +64,18 @@ install:
 
   ##################################################
   # Install packages for docs.
-  # DRi#2616: internet outages have caused builds to spin for tens of minutes, only to fail
-  # due to chocolatey and fig2dev download errors. We instead ignore errors here and
-  # continue to build and test anyway.
-  - choco install --timeout 600 -y doxygen.portable imagemagick.tool ghostscript || true
-  # The ghostscript package is an install that doesn't add to PATH.
-  # FIXME i#2145: fig2dev is failing to launch ghostscript, so for now
-  # we're disabling to get the initial AppVeyor to be green.
-  #- set PATH=C:\Program Files\gs\gs9.20\bin;%PATH%
-  # Install fig2dev. DRi#2616: we ignore fig2dev download errors.
-  - appveyor DownloadFile http://www.winfig.com/WinFIG-Dateien/WinFIG62.zip || true
-  - 7z x WinFIG62.zip > nul || true
-  - set PATH=c:\projects\install\WinFig;%PATH%
+  # DRi#4000: choco fails to download doxygen.portable so we install ourselves:
+  - appveyor DownloadFile http://doxygen.nl/files/doxygen-1.8.17.windows.x64.bin.zip
+  - 7z x doxygen-1.8.17.windows.x64.bin.zip -oc:\projects\install\doxygen > nul
+  - set PATH=c:\projects\install\doxygen;%PATH%
+
+  # AppVeyor has WiX installed but it is not on the PATH.
+  # We first print out all versions to make it easier to update this if
+  # the version changes.
+  - dir "c:\Program Files (x86)\WiX Toolset"*
+  # DRi#4108: Work around runsuite_common_pre.cmake appending x64 to bin here.
+  - ps: $wixpath=& c:\cygwin64\bin\cygpath -d 'C:\Program Files (x86)\WiX Toolset v3.11\bin'
+  - ps: $env:PATH="$wixpath;$env:PATH"
 
   ##################################################
   # XXX i#2145: point at Qt5 for testing drheapstat visualizer.
@@ -96,13 +96,15 @@ build_script:
 # Automated deployment of builds to GitHub Releases.
 # We rely on a Travis cron job to push a tag to the repo which then
 # triggers this deployment.
-# We disable test running for these package builds in runsuite.cmake by
-# looking for $APPVEYOR_REPO_TAG=="true".
-# XXX i#1967: build an .msi package as well.
+# We switch to package.cmake for these builds in runsuite_wrapper.pl by looking
+# for $APPVEYOR_REPO_TAG=="true".
 artifacts:
-  - path: 'build\build_*\DrMemory*.zip'
+  - path: 'build\DrMemory*.zip'
     name: DrMem.zip
     type: zip
+  - path: 'build\DrMemory*.msi'
+    name: DrMem.msi
+    type: msi
 deploy:
   provider: GitHub
   auth_token:
@@ -114,7 +116,7 @@ deploy:
   force_update: true
   # Using the default "release:" name (the tag) to match Travis.
   # This description replaces the one provided by Travis.
-  description: 'Auto-generated periodic build (Appveyor build $(appveyor_build_version)).'
+  description: 'Auto-generated periodic build.'
   on:
     branch: master
     appveyor_repo_tag: true

.travis.yml

@@ -45,55 +45,71 @@ language:
 # with exludes so we can use conditional builds (plus it's clearer this way).
 jobs:
   include:
-    - os: linux
+    #######################################################################
+    # Regular test jobs
+    - if: type != cron
+      os: linux
       compiler: gcc
-      env: DEPLOY=yes
-    - os: linux
+    - if: type != cron
+      os: linux
+      compiler: clang
+    - if: type != cron
+      os: osx
       compiler: clang
-    - os: osx
-      # gcc on Travis claims to not be CMAKE_COMPILER_IS_GNUCC so we only run clang.
+    #######################################################################
+    # Package jobs
+    - if: type = cron
+      os: linux
+      compiler: gcc
+      env: DEPLOY=yes
+    - if: type = cron
+      os: osx
       compiler: clang
       env: DEPLOY=yes
-      # XXX: DRi#2764: Travis OSX resources are over-subscribed and it can take
-      # hours to get an OSX machine, so we skip running PR's for now.
-      if: type != pull_request
 
 # For C/C++ there is no default install, so we set "install", not "before_install".
 install:
   - uname -a
-  # FIXME: remove the "brew update" step once Travis fixes their Mac VM's
+  # XXX: Remove the "brew update" step once Travis fixes their Mac VM's
   # on 11/15/17.  Xref https://github.com/travis-ci/travis-ci/issues/8552.
   - if [[ "`uname`" == "Darwin" ]]; then brew update; brew install nasm; fi
-  # ImageMagick is present but these are not:
-  - if [[ "`uname`" == "Linux" ]]; then sudo apt-get -y install g++-multilib ghostscript doxygen transfig; fi
+  - if [[ "`uname`" == "Linux" ]]; then sudo apt-get -y install g++-multilib doxygen; fi
 
 script:
   - tests/runsuite_wrapper.pl travis
 
-# We disable test running for these package builds in runsuite.cmake by
-# looking for $TRAVIS_EVENT_TYPE=="cron".
-# Longer-term we may want to use package.cmake instead and even make official
-# builds on Travis (DRi#2861).
+# We switch to package.cmake for these builds in runsuite_wrapper.pl by looking
+# for $TRAVIS_EVENT_TYPE=="cron".
 before_deploy:
   - git config --local user.name "Travis Auto-Tag"
   - git config --local user.email "drmemory-devs@googlegroups.com"
   # XXX: for now we duplicate this version number here with CMakeLists.txt.
   # We should find a way to share (xref DRi#1565).
-  # We support setting TAG_SUFFIX on triggered builds so we can have
-  # multiple unique tags in one day (the patchlevel here is the day number).
-  - export GIT_TAG="cronbuild-2.3.$((`git log -n 1 --format=%ct` / (60*60*24)))${TAG_SUFFIX}"
-  - git tag $GIT_TAG -a -m "Travis auto-generated tag for build $TRAVIS_BUILD_NUMBER."
+  # We support setting VERSION_NUMBER for manual builds to override all
+  # parts of the version.  If a build is included (leading dash and number
+  # at the end), it will be parsed and passed to package.cmake.  We only
+  # use a non-zero build number when making multiple manual builds in one day.
+  - >
+      if test -z "${VERSION_NUMBER}"; then
+          export GIT_TAG="cronbuild-2.3.$((`git log -n 1 --format=%ct` / (60*60*24)))"
+      else
+          export GIT_TAG="release_${VERSION_NUMBER}"
+      fi
+  # We handle races among our 4 package jobs by ignoring failure here.
+  # XXX: That could mask a real failure: we could try to distinguish the
+  # type of error.
+  - (git tag $GIT_TAG -a -m "Travis auto-generated tag for build $TRAVIS_BUILD_NUMBER." || true)
 deploy:
   provider: releases
   api_key:
     secure: HDMWsqw+DqxPg1lhy9pk23EyHI5PdTBo3/SA//h41tIKi2+uEtvoETidRXa/ePkEymVRGyzF7F5RVNFeKu4gM3taxa7JUbOc1OgC8o7oRVfYXZd1qALdDf03EyYAtw3iTeLKlOBSR1cnRZcoOZ6Ja9E6OZC9qjAPrXywNzZkkv4=
   file_glob: true
-  file: "build*/DrMemory*.tar.gz"
+  file: "DrMemory*.tar.gz"
   skip_cleanup: true
   # The name must just be the tag in order to match Appveyor.
   name: $GIT_TAG
   # This body is clobbered by Appveyor.
-  body: "Auto-generated periodic build (Travis build $TRAVIS_BUILD_NUMBER)."
+  body: "Auto-generated periodic build."
   on:
     repo: DynamoRIO/drmemory
     branch: master

CMakeLists.txt

@@ -2374,8 +2374,14 @@ endif (WIN32 AND NOT BUILDING_SUB_PACKAGE)
 # CPACK_TEMPORARY_PACKAGE_FILE_NAME if I hardcode the exentsion: but maybe
 # having the full version in the base dir is a good thing, though I'm not
 # sure about the caps.
+# We omit the -NN suffix for the build number if it is zero.
+if ("${TOOL_BUILD_NUMBER}" STREQUAL "0")
+  set(PACKAGE_SUFFIX "")
+else ()
+  set(PACKAGE_SUFFIX "-${TOOL_BUILD_NUMBER}")
+endif ()
 set(CPACK_PACKAGE_FILE_NAME
-  "${toolname_cap}-${CPACK_SYSTEM_NAME}-${CPACK_PACKAGE_VERSION}-${TOOL_BUILD_NUMBER}")
+  "${toolname_cap}-${CPACK_SYSTEM_NAME}-${CPACK_PACKAGE_VERSION}${PACKAGE_SUFFIX}")
 set(CPACK_SOURCE_PACKAGE_FILE_NAME
   "${toolname_cap}-${CPACK_PACKAGE_VERSION}-${TOOL_BUILD_NUMBER}-Source")
 

docs/CMake_doxyfile.cmake

@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2011-2016 Google, Inc.  All rights reserved.
+# Copyright (c) 2011-2020 Google, Inc.  All rights reserved.
 # Copyright (c) 2009-2010 VMware, Inc.  All rights reserved.
 # **********************************************************
 

package.cmake

@@ -48,6 +48,7 @@ set(arg_preload "")    # cmake file to include prior to each 32-bit build
 set(arg_preload64 "")  # cmake file to include prior to each 64-bit build
 set(arg_use_nmake OFF) # use nmake even if gnu make is present
 set(arg_cpackappend "")# string to append to CPackConfig.cmake before packaging
+set(arg_travis OFF)
 
 foreach (arg ${CTEST_SCRIPT_ARG})
   if (${arg} MATCHES "^build=")
@@ -84,6 +85,9 @@ foreach (arg ${CTEST_SCRIPT_ARG})
       ${arg} MATCHES "^drmemory_only")
     set(arg_drmem_only ON)
   endif ()
+  if (${arg} STREQUAL "travis")
+    set(arg_travis ON)
+  endif ()
 endforeach (arg)
 
 if ("${arg_build}" STREQUAL "")
@@ -198,8 +202,12 @@ endif ()
 
 if (NOT arg_sub_package)
   set(build_package ON)
-  # some packagers request an official source tarball, so we create one (i#1287):
-  set(build_source_package ON)
+  # Some packagers request an official source tarball, so we create one (i#1287).
+  # However, this fails on Appveyor for as-yet-undiagnosed reasons (i#2255),
+  # and Github supplies its own source tarball, so we disable there.
+  if (NOT arg_travis)
+    set(build_source_package ON)
+  endif ()
   include("${runsuite_include_path}/runsuite_common_post.cmake")
 
   # copy the final archive into cur dir

tests/runsuite.cmake

@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2010-2018 Google, Inc.  All rights reserved.
+# Copyright (c) 2010-2020 Google, Inc.  All rights reserved.
 # Copyright (c) 2009-2010 VMware, Inc.  All rights reserved.
 # **********************************************************
 
@@ -350,7 +350,7 @@ if (UNIX AND ARCH_IS_X86)
   set(ARCH_IS_X86 ON)
 endif (UNIX AND ARCH_IS_X86)
 
-if (NOT arg_vmk_only AND NOT arg_already_built)
+if (NOT arg_vmk_only AND NOT arg_already_built AND NOT arg_travis)
   set(build_package ON)
 else ()
   set(build_package OFF)

tests/runsuite_wrapper.pl

@@ -48,6 +48,15 @@
     }
 }
 
+my $osdir = $mydir;
+if ($^O eq 'cygwin') {
+    # CMake is native Windows so pass it a Windows path.
+    # We use the full path to cygpath as git's cygpath is earlier on
+    # the PATH for AppVeyor and it fails.
+    $osdir = `/usr/bin/cygpath -wi \"$mydir\"`;
+    chomp $osdir;
+}
+
 # We have no way to access the log files, so we use -VV to ensure
 # we can diagnose failures.
 # We tee to stdout to provide incremental output and avoid the 10-min
@@ -63,19 +72,41 @@
         $res .= $_;
     }
     close(CHILD);
-} else {
-    if ($^O eq 'cygwin') {
-        # CMake is native Windows so pass it a Windows path.
-        # We use the full path to cygpath as git's cygpath is earlier on
-        # the PATH for AppVeyor and it fails.
-        $mydir = `/usr/bin/cygpath -wi \"$mydir\"`;
-        chomp $mydir;
+} elsif ($ENV{'TRAVIS_EVENT_TYPE'} eq 'cron' ||
+         $ENV{'APPVEYOR_REPO_TAG'} eq 'true') {
+    # A package build.
+    my $build = "0";
+    # We trigger by setting VERSION_NUMBER in Travis.
+    # That sets a tag and we propagate the name into the Appveyor build from the tag:
+    if ($ENV{'APPVEYOR_REPO_TAG_NAME'} =~ /release_(.*)/) {
+        $ENV{'VERSION_NUMBER'} = $1;
     }
+    if ($ENV{'VERSION_NUMBER'} =~ /-(\d+)$/) {
+        $build = $1;
+    }
+    if ($args eq '') {
+        $args = ",";
+    } else {
+        $args .= ";";
+    }
+    $args .= "drmem_only;build=${build}";
+    if ($^O eq 'darwin' || $^O eq 'MacOS') {
+        $args .= ";64_only";
+    }
+    if ($ENV{'VERSION_NUMBER'} =~ /^(\d+\.\d+\.\d+)/) {
+        my $version = $1;
+        $args .= ";version=${version}";
+    }
+    my $cmd = "ctest -VV -S \"${osdir}/../package.cmake${args}\"";
+    print "Running ${cmd}\n";
+    system("${cmd} 2>&1");
+    exit 0;
+} else {
     # To shrink the log sizes and make Travis and Appveyor error pages easier
     # to work with we omit a second V and instead use --output-on-failure.
     # We rely on runsuite_common_post.cmake extracting configure and build error
     # details from the xml files, as they don't show up with one V.
-    system("ctest --output-on-failure -V -S \"${mydir}/runsuite.cmake${args}\" 2>&1");
+    system("ctest --output-on-failure -V -S \"${osdir}/runsuite.cmake${args}\" 2>&1");
     exit 0;
 }
 
@@ -100,6 +131,10 @@
         $fail = 1;
         $should_print = 1;
         $name = "diff pre-commit checks";
+    } elsif ($line =~ /^FAILED: CMakeFiles\/package/) {
+        $fail = 1;
+        $should_print = 1;
+        $name = "packaging step";
     }
     if ($fail && $is_CI  && $line =~ /tests failed/) {
         my $is_32 = $line =~ /-32/;
@@ -186,6 +221,23 @@
         print "\n====> FAILURE in $name <====\n";
     }
     print "$line\n" if ($should_print);
+    if ($line =~ /Please check '([^']+)' for errors/) {
+        my $log = $1;
+        if ($^O eq 'cygwin') {
+            $log = `/usr/bin/cygpath -u \"$log\"`;
+        }
+        chomp $log;
+        if (open(LOG, "< $log")) {
+            print "\n\n----------------- START $log -----------\n";
+            while (<LOG>) {
+                print $_;
+            }
+            print "\n----------------- END $log -----------\n\n";
+        } else {
+            print "Failed to open $log\n";
+        }
+        close(LOG);
+    }
 }
 if (!$should_print) {
     print "Error: RESULTS line not found\n";