Improve Windows build.

3rdparty/fetch_and_build.sh

@@ -1,16 +1,12 @@
 #!/bin/sh
 
 sudo apt-get install gcc-mingw-w64
-sudo apt-get install gcc-arm-linux-gnueabihf
 
 if [ ! -d openssl-1.0.2-winapi ]; then
  echo
  echo Fetching OpenSSL
  echo
  git clone https://github.com/openssl/openssl.git openssl-1.0.2-winapi
- if [ ! -d openssl-1.0.2-arm ]; then
- git clone openssl-1.0.2-winapi openssl-1.0.2-arm
- fi
 fi
 
 cd openssl-1.0.2-winapi
@@ -25,34 +21,15 @@ if [ ! -f ssleay32.dll ]; then
 fi
 cd ..
 
-exit 0
-
-cd openssl-1.0.2-arm
-if [ ! -f libssl.a ]; then
- echo
- echo Building OpenSSL arm32
- echo
-
- git checkout OpenSSL_1_0_2-stable
- ./Configure linux-generic32 shared --cross-compile-prefix=arm-linux-gnueabihf-
- make
-fi
-cd ..
-
-if [ ! -d zlib-1.2.11-winapi ]; then
+if [ ! -d zlib-1.3-winapi ]; then
  echo
  echo Fetching zlib
  echo
- wget https://zlib.net/zlib-1.2.11.tar.gz && tar xzf zlib-1.2.11.tar.gz
- mv zlib-1.2.11 zlib-1.2.11-winapi
-fi
-
-if [ ! -d zlib-1.2.11-arm ]; then
- tar xzf zlib-1.2.11.tar.gz
- mv zlib-1.2.11 zlib-1.2.11-arm
+ wget https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz && tar xzf zlib-1.3.tar.gz
+ mv zlib-1.3 zlib-1.3-winapi
 fi
 
-cd zlib-1.2.11-winapi
+cd zlib-1.3-winapi
 if [ ! -f zlib1.dll ]; then
  echo
  echo Building zlib w64
@@ -61,27 +38,6 @@ if [ ! -f zlib1.dll ]; then
 fi
 cd ..
 
-cd zlib-1.2.11-arm
-if [ ! -f libz.a ]; then
- echo
- echo Building zlib arm32
- echo
- CROSS=arm-linux-gnueabihf- CC=${CROSS}gcc LD=${CROSS}ld AS=${CROSS}as ./configure
- make
-fi
-cd ..
-
-if [ ! -d libfuse-arm ]; then
- echo
- echo Fetching libfuse arm
- echo
- wget http://http.us.debian.org/debian/pool/main/f/fuse/libfuse-dev_2.9.0-2+deb7u2_armhf.deb
- mkdir -p libfuse-arm
- cd libfuse-arm
- ar x ../libfuse-dev_2.9.0-2+deb7u2_armhf.deb
- tar xzf data.tar.gz
-fi
-
 if [ ! -d winfsp ]; then
  echo
  echo Fetching Winfsp
@@ -94,9 +50,6 @@ if [ ! -d librsync-winapi ]; then
  echo Fetching RSync
  echo
  git clone https://github.com/librsync/librsync.git librsync-winapi
- if [ ! -d librsync-arm ]; then
- git clone librsync-winapi librsync-arm
- fi
 fi
 
 cd librsync-winapi

README.md

@@ -112,7 +112,7 @@ the cloud, before you stuff your laptop in your bag.
 The store created a _point in time_, which is defined to be the modify
 timestamp of the most recently changed file/directory within the
 _origin_ directory that was backed up. (I.e. it is not the time when
-the store was initiated.)
+the store was initiated.) Note that the timestamp is truncated to microseconds.
 
 `work:` a beak _rule_ that you have created to backup the _origin_ directory /home/you/Work to
 the local backup /home/you/Work/.beak/local and then to the remote cloud storage location s3_work_crypt:
@@ -382,13 +382,13 @@ file and the hash of the meta-data of the files inside the archive file. Like th
 
 `beak_s_1575491997.351991_2dff0195a0a468e1eab4cffe0472956a5c8374cc7caa1c2ae08af5564f974cdf_1-1_16384_20000.tar`
 
-`beak_s_seconds.nanos_metadatahash_partnr_numparts_size_paddedsize.tar`
+`beak_s_seconds.micros_metadatahash_partnr_numparts_size_paddedsize.tar`
 
 The index file, similarily named, but uses the most recently modify timestamp
 of any of the archive files it indexes _and_ the modify timestamp of the containing directory.
 The hash is the hash of the all meta-data hashes (for the tars) pointed to by this index file.
 
-`beak_z_--seconds---.--nanos--_----------metadata-hash-----------------------------------------_1-1_--size---_--paddedsize---0.gz`
+`beak_z_--seconds---.--micros--_----------metadata-hash-----------------------------------------_1-1_--size---_--paddedsize---0.gz`
 
 Assuming that your _origin_ directory `/home/you/Work/` above contained `rclonesrc/` and `gamesrc/`
 and you made the backup to `/home/you/Backup`. Now modify a file in `gamesrc/` and do another backup:
@@ -510,7 +510,6 @@ beak pack {<rule>|<storage>}
 Hosts supported:
 * x86_64-pc-linux-gnu
 * x86_64-w64-mingw32
-* arm-linux-gnueabihf
 * x86_64-apple-darwin18.2.0
 
 To have beak print detailed debug information do: `export BEAK_DEBUG_hardlinks=true`
@@ -528,13 +527,10 @@ You can have multiple configurations enabled at the same time.
 * `make debug` builds debug for all configured hosts.
 * `make debug linux64` builds only debug for gnu/linux x86_64 hosts.
 * `make debug winapi64` builds only debug for winapi 64 bit hosts.
-* `make debug arm32` builds only debug for gnu/linux arm 32 bit hosts.
 
 `./configure`
 
-`./configure --host=x86_64-w64-mingw32 --with-zlib=3rdparty/zlib-1.2.11-winapi --with-openssl=3rdparty/openssl-1.0.2-winapi`
-
-`./configure --host=arm-linux-gnueabihf --with-fuse=3rdparty/libfuse-arm/usr --with-openssl=3rdparty/openssl-1.0.2-arm --with-zlib=3rdparty/zlib-1.2.11-arm`
+`./configure --host=x86_64-w64-mingw32 --disable-media --with-zlib=3rdparty/zlib-1.3-winapi --with-openssl=3rdparty/openssl-1.0.2-winapi --with-librsync=3rdparty/librsync-winapi`
 
 ### Compiling on Darwin
 

configure

@@ -4594,7 +4594,7 @@ printf "%s\n" "found" >&6; }
 else
  with_openssl=$(realpath ${with_openssl})
  OPENSSL_CFLAGS="-I${with_openssl}/include -DOPENSSL_API_COMPAT=0x10100000L"
- OPENSSL_LIBS="${with_openssl}/lib/libcrypto.a"
+ OPENSSL_LIBS="${with_openssl}/libcrypto.a"
  if ! test -r ${with_openssl}/include/openssl/sha.h
  then
  as_fn_error $? "Expected openssl/sha.h to be find!" "$LINENO" 5
@@ -4721,6 +4721,8 @@ fi
 printf "%s\n" "found" >&6; }
  fi
 
+else
+ echo "media disabled"
 fi
 
 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sizeof time_t" >&5

configure.ac

@@ -1,4 +1,4 @@
-# Copyright (C) 2017 Fredrik Öhrström
+# Copyright (C) 2017-2023 Fredrik Öhrström
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -191,7 +191,7 @@ if test "x${with_openssl}" = "x"; then
 else
  with_openssl=$(realpath ${with_openssl})
  OPENSSL_CFLAGS="-I${with_openssl}/include -DOPENSSL_API_COMPAT=0x10100000L"
- OPENSSL_LIBS="${with_openssl}/lib/libcrypto.a"
+ OPENSSL_LIBS="${with_openssl}/libcrypto.a"
  if ! test -r ${with_openssl}/include/openssl/sha.h
  then
  AC_MSG_ERROR([Expected openssl/sha.h to be find!])
@@ -284,6 +284,8 @@ if test "${ENABLE_MEDIA}" = "yes"; then
  AC_MSG_RESULT([found])
  fi
 
+else
+ echo "media disabled"
 fi
 
 AC_MSG_CHECKING(for sizeof time_t)

make/Main.mk

@@ -44,14 +44,31 @@ $(shell mkdir -p $(OUTPUT_ROOT)/$(TYPE))
 
 VERBOSE?=@
 
+ifeq ($(PLATFORM),WINAPI)
+STRIP_COMMAND:=true
+endif
+
+MEDIA_SOURCES:=beak_importmedia.cc beak_servemedia.cc beak_indexmedia.cc media.cc
+MEDIA_SOURCES:=$(addprefix $(SRC_ROOT)/src/,$(MEDIA_SOURCES))
+NO_MEDIA_SOURCES:=no_beak_importmedia.cc no_beak_servemedia.cc no_beak_indexmedia.cc no_media.cc
+NO_MEDIA_SOURCES:=$(addprefix $(SRC_ROOT)/src/,$(NO_MEDIA_SOURCES))
+
 WINAPI_SOURCES:=$(filter-out %posix.cc, $(wildcard $(SRC_ROOT)/src/*.cc))
 ifeq ($(ENABLE_FUSE),yes)
 WINAPI_SOURCES:=$(filter-out %no_fuse.cc,$(WINAPI_SOURCES))
 endif
 
+WINAPI_SOURCES:=$(filter-out %media.cc,$(WINAPI_SOURCES))
+
 WINAPI_OBJS:=\
  $(patsubst %.cc,%.o,$(subst $(SRC_ROOT)/src,$(OUTPUT_ROOT)/$(TYPE),$(WINAPI_SOURCES)))
 
+WINAPI_MEDIA_OBJS:=\
+ $(patsubst %.cc,%.o,$(subst $(SRC_ROOT)/src,$(OUTPUT_ROOT)/$(TYPE),$(MEDIA_SOURCES)))
+
+WINAPI_NO_MEDIA_OBJS:=\
+ $(patsubst %.cc,%.o,$(subst $(SRC_ROOT)/src,$(OUTPUT_ROOT)/$(TYPE),$(NO_MEDIA_SOURCES)))
+
 WINAPI_BEAK_OBJS:=\
  $(filter-out %testinternals.o,$(WINAPI_OBJS))
 
@@ -60,6 +77,7 @@ $(OUTPUT_ROOT)/$(TYPE)/libgcc_s_seh-1.dll \
 $(OUTPUT_ROOT)/$(TYPE)/libstdc++-6.dll \
 $(OUTPUT_ROOT)/$(TYPE)/libwinpthread-1.dll
 
+
 WINAPI_TESTINTERNALS_OBJS:=\
  $(filter-out %main.o,$(WINAPI_OBJS))
 
@@ -71,11 +89,6 @@ endif
 
 POSIX_SOURCES:=$(filter-out %media.cc,$(POSIX_SOURCES))
 
-MEDIA_SOURCES:=beak_importmedia.cc beak_servemedia.cc beak_indexmedia.cc media.cc
-MEDIA_SOURCES:=$(addprefix $(SRC_ROOT)/src/,$(MEDIA_SOURCES))
-NO_MEDIA_SOURCES:=no_beak_importmedia.cc no_beak_servemedia.cc no_beak_indexmedia.cc no_media.cc
-NO_MEDIA_SOURCES:=$(addprefix $(SRC_ROOT)/src/,$(NO_MEDIA_SOURCES))
-
 POSIX_OBJS:=\
  $(patsubst %.cc,%.o,$(subst $(SRC_ROOT)/src,$(OUTPUT_ROOT)/$(TYPE),$(POSIX_SOURCES)))
 
@@ -115,8 +128,8 @@ $(OUTPUT_ROOT)/$(TYPE)/fileinfo.o: $(OUTPUT_ROOT)/generated_filetypes.h
 
 $(OUTPUT_ROOT)/$(TYPE)/%.o: $(SRC_ROOT)/src/%.cc
  @echo Compiling $(TYPE) $(CONF_MNEMONIC) $$(basename $<)
- $(VERBOSE)$(CXX) $(CXXFLAGS_$(TYPE)) $(CXXFLAGS) -I$(OUTPUT_ROOT) -I$(BUILD_ROOT) -MMD $< -c -o $@
- $(VERBOSE)$(CXX) -E $(CXXFLAGS_$(TYPE)) $(CXXFLAGS) -I$(OUTPUT_ROOT) -I$(BUILD_ROOT) -MMD $< -c > $@.source
+ $(VERBOSE)$(CXX) -I$(OUTPUT_ROOT) -I$(BUILD_ROOT) $(CXXFLAGS_$(TYPE)) $(CXXFLAGS) -MMD $< -c -o $@
+ $(VERBOSE)$(CXX) -E  -I$(OUTPUT_ROOT) -I$(BUILD_ROOT) $(CXXFLAGS_$(TYPE)) $(CXXFLAGS) -MMD $< -c > $@.source
 
 $(OUTPUT_ROOT)/$(TYPE)/beak-media: $(BEAK_OBJS) $(BEAK_MEDIA_OBJS)
  @echo Linking $(TYPE) $(CONF_MNEMONIC) $@
@@ -125,6 +138,7 @@ $(OUTPUT_ROOT)/$(TYPE)/beak-media: $(BEAK_OBJS) $(BEAK_MEDIA_OBJS)
  $(VERBOSE)$(STRIP_COMMAND) $@
  @echo Done linking $(TYPE) $(CONF_MNEMONIC) $@
 
+$(info BEAK_NO_MEDIA_OBJS=$(BEAK_NO_MEDIA_OBJS))
 $(OUTPUT_ROOT)/$(TYPE)/beak: $(BEAK_OBJS) $(BEAK_NO_MEDIA_OBJS)
  @echo Linking $(TYPE) $(CONF_MNEMONIC) $@
  $(VERBOSE)$(CXX) -o $@ $(LDFLAGS_$(TYPE)) $(LDFLAGS) $(BEAK_OBJS) $(BEAK_NO_MEDIA_OBJS) \
@@ -139,11 +153,11 @@ $(OUTPUT_ROOT)/$(TYPE)/testinternals: $(TESTINTERNALS_OBJS) $(BEAK_NO_MEDIA_OBJS
  $(VERBOSE)$(STRIP_COMMAND) $@
  @echo Done linking $(TYPE) $(CONF_MNEMONIC) $@
 
-$(OUTPUT_ROOT)/$(TYPE)/libgcc_s_seh-1.dll: /usr/lib/gcc/x86_64-w64-mingw32/5.3-win32/libgcc_s_seh-1.dll
- cp /usr/lib/gcc/x86_64-w64-mingw32/5.3-win32/libgcc_s_seh-1.dll $@
+$(OUTPUT_ROOT)/$(TYPE)/libgcc_s_seh-1.dll: /usr/lib/gcc/x86_64-w64-mingw32/10-win32/libgcc_s_seh-1.dll
+ cp /usr/lib/gcc/x86_64-w64-mingw32/10-win32/libgcc_s_seh-1.dll $@
 
-$(OUTPUT_ROOT)/$(TYPE)/libstdc++-6.dll: /usr/lib/gcc/x86_64-w64-mingw32/5.3-win32/libstdc++-6.dll
- cp /usr/lib/gcc/x86_64-w64-mingw32/5.3-win32/libstdc++-6.dll $@
+$(OUTPUT_ROOT)/$(TYPE)/libstdc++-6.dll: /usr/lib/gcc/x86_64-w64-mingw32/10-win32/libstdc++-6.dll
+ cp /usr/lib/gcc/x86_64-w64-mingw32/10-win32/libstdc++-6.dll $@
 
 $(OUTPUT_ROOT)/$(TYPE)/libwinpthread-1.dll: /usr/x86_64-w64-mingw32/lib/libwinpthread-1.dll
  cp $< $@

src/filesystem_winapi.cc

@@ -120,6 +120,7 @@ struct FileSystemImplementationWinapi : FileSystem
  RC addWatch(Path *dir);
  int endWatch();
  FILE *openAsFILE(Path *p, const char *mode);
+ void allowAccessTimeUpdates();
 
  FileSystemImplementationWinapi() : FileSystem("FileSystemImplementationWinapi") {}
 
@@ -496,3 +497,7 @@ FILE *FileSystemImplementationWinapi::openAsFILE(Path *p, const char *mode)
 {
  return fopen(p->c_str(), mode);
 }
+
+void FileSystemImplementationWinapi::allowAccessTimeUpdates()
+{
+}

src/main.cc

@@ -25,7 +25,6 @@
 #include "storagetool.h"
 #include "system.h"
 
-#include<sys/resource.h>
 #include <stdio.h>
 #include <unistd.h>
 
@@ -46,41 +45,19 @@ int main(int argc, char *argv[])
  }
 }
 
-int setStackSize()
-{
- const rlim_t kStackSize = 32 * 1024 * 1024; // min stack size = 32 MB
- struct rlimit rl;
- int result;
-
- result = getrlimit(RLIMIT_STACK, &rl);
- if (result == 0)
- {
- if (rl.rlim_cur < kStackSize)
- {
- rl.rlim_cur = kStackSize;
- result = setrlimit(RLIMIT_STACK, &rl);
- if (result != 0)
- {
- fprintf(stderr, "setrlimit returned result = %d\n", result);
- }
- }
- }
-
- // ...
-
- return 0;
-}
 
 int run(int argc, char *argv[])
 {
  RC rc = RC::OK;
 
- setStackSize();
-
  ::captureStartTime();
 
  // First create the OS interface to invoke external commands like rclone and rsync.
  auto sys = newSystem();
+
+ // Increase the stack size.
+ sys->setStackSize();
+
  // Next create the interface to the local file system where we find:
  // the orgin files and directories, the beak configuration file, the rclone configuration file,
  // and the temporary/cache files.

src/system.h

@@ -63,6 +63,8 @@ struct System
  virtual RC umount(ptr<FuseMount> fuse_mount) = 0;
  // The current user running the beak software.
  virtual std::string userName() = 0;
+ // Increase the stack size.
+ virtual void setStackSize() = 0;
 
  virtual ~System() = default;
 };

src/system_posix.cc

@@ -25,6 +25,7 @@
 #include <pwd.h>
 #include <sys/errno.h>
 #include <sys/types.h>
+#include <sys/resource.h>
 #ifdef OSX64
 #include <signal.h>
 #include <sys/wait.h>
@@ -229,6 +230,8 @@ struct SystemImplementation : System
  SystemImplementation();
  ~SystemImplementation() = default;
 
+ void setStackSize();
+
  private:
 
  pid_t running_shell_pid_ {};
@@ -561,3 +564,26 @@ string SystemImplementation::userName()
 {
  return user_name_;
 }
+
+void SystemImplementation::setStackSize()
+{
+ const rlim_t kStackSize = 32 * 1024 * 1024; // min stack size = 32 MB
+ struct rlimit rl;
+ int result;
+
+ result = getrlimit(RLIMIT_STACK, &rl);
+ if (result == 0)
+ {
+ if (rl.rlim_cur < kStackSize)
+ {
+ rl.rlim_cur = kStackSize;
+ result = setrlimit(RLIMIT_STACK, &rl);
+ if (result != 0)
+ {
+ fprintf(stderr, "setrlimit returned result = %d\n", result);
+ }
+ }
+ }
+
+ // ...
+}