diff --git a/3rdparty/fetch_and_build.sh b/3rdparty/fetch_and_build.sh index a4cee05..1a99b8c 100755 --- a/3rdparty/fetch_and_build.sh +++ b/3rdparty/fetch_and_build.sh @@ -1,16 +1,12 @@ #!/bin/sh sudo apt-get install gcc-mingw-w64 -sudo apt-get install gcc-arm-linux-gnueabihf if [ ! -d openssl-1.0.2-winapi ]; then echo echo Fetching OpenSSL echo git clone https://github.com/openssl/openssl.git openssl-1.0.2-winapi - if [ ! -d openssl-1.0.2-arm ]; then - git clone openssl-1.0.2-winapi openssl-1.0.2-arm - fi fi cd openssl-1.0.2-winapi @@ -25,34 +21,15 @@ if [ ! -f ssleay32.dll ]; then fi cd .. -exit 0 - -cd openssl-1.0.2-arm -if [ ! -f libssl.a ]; then - echo - echo Building OpenSSL arm32 - echo - - git checkout OpenSSL_1_0_2-stable - ./Configure linux-generic32 shared --cross-compile-prefix=arm-linux-gnueabihf- - make -fi -cd .. - -if [ ! -d zlib-1.2.11-winapi ]; then +if [ ! -d zlib-1.3-winapi ]; then echo echo Fetching zlib echo - wget https://zlib.net/zlib-1.2.11.tar.gz && tar xzf zlib-1.2.11.tar.gz - mv zlib-1.2.11 zlib-1.2.11-winapi -fi - -if [ ! -d zlib-1.2.11-arm ]; then - tar xzf zlib-1.2.11.tar.gz - mv zlib-1.2.11 zlib-1.2.11-arm + wget https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz && tar xzf zlib-1.3.tar.gz + mv zlib-1.3 zlib-1.3-winapi fi -cd zlib-1.2.11-winapi +cd zlib-1.3-winapi if [ ! -f zlib1.dll ]; then echo echo Building zlib w64 @@ -61,27 +38,6 @@ if [ ! -f zlib1.dll ]; then fi cd .. -cd zlib-1.2.11-arm -if [ ! -f libz.a ]; then - echo - echo Building zlib arm32 - echo - CROSS=arm-linux-gnueabihf- CC=${CROSS}gcc LD=${CROSS}ld AS=${CROSS}as ./configure - make -fi -cd .. - -if [ ! -d libfuse-arm ]; then - echo - echo Fetching libfuse arm - echo - wget http://http.us.debian.org/debian/pool/main/f/fuse/libfuse-dev_2.9.0-2+deb7u2_armhf.deb - mkdir -p libfuse-arm - cd libfuse-arm - ar x ../libfuse-dev_2.9.0-2+deb7u2_armhf.deb - tar xzf data.tar.gz -fi - if [ ! -d winfsp ]; then echo echo Fetching Winfsp @@ -94,9 +50,6 @@ if [ ! -d librsync-winapi ]; then echo Fetching RSync echo git clone https://github.com/librsync/librsync.git librsync-winapi - if [ ! -d librsync-arm ]; then - git clone librsync-winapi librsync-arm - fi fi cd librsync-winapi diff --git a/README.md b/README.md index a0b2c29..df491d0 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ the cloud, before you stuff your laptop in your bag. The store created a _point in time_, which is defined to be the modify timestamp of the most recently changed file/directory within the _origin_ directory that was backed up. (I.e. it is not the time when -the store was initiated.) +the store was initiated.) Note that the timestamp is truncated to microseconds. `work:` a beak _rule_ that you have created to backup the _origin_ directory /home/you/Work to the local backup /home/you/Work/.beak/local and then to the remote cloud storage location s3_work_crypt: @@ -382,13 +382,13 @@ file and the hash of the meta-data of the files inside the archive file. Like th `beak_s_1575491997.351991_2dff0195a0a468e1eab4cffe0472956a5c8374cc7caa1c2ae08af5564f974cdf_1-1_16384_20000.tar` -`beak_s_seconds.nanos_metadatahash_partnr_numparts_size_paddedsize.tar` +`beak_s_seconds.micros_metadatahash_partnr_numparts_size_paddedsize.tar` The index file, similarily named, but uses the most recently modify timestamp of any of the archive files it indexes _and_ the modify timestamp of the containing directory. The hash is the hash of the all meta-data hashes (for the tars) pointed to by this index file. -`beak_z_--seconds---.--nanos--_----------metadata-hash-----------------------------------------_1-1_--size---_--paddedsize---0.gz` +`beak_z_--seconds---.--micros--_----------metadata-hash-----------------------------------------_1-1_--size---_--paddedsize---0.gz` Assuming that your _origin_ directory `/home/you/Work/` above contained `rclonesrc/` and `gamesrc/` and you made the backup to `/home/you/Backup`. Now modify a file in `gamesrc/` and do another backup: @@ -510,7 +510,6 @@ beak pack {|} Hosts supported: * x86_64-pc-linux-gnu * x86_64-w64-mingw32 -* arm-linux-gnueabihf * x86_64-apple-darwin18.2.0 To have beak print detailed debug information do: `export BEAK_DEBUG_hardlinks=true` @@ -528,13 +527,10 @@ You can have multiple configurations enabled at the same time. * `make debug` builds debug for all configured hosts. * `make debug linux64` builds only debug for gnu/linux x86_64 hosts. * `make debug winapi64` builds only debug for winapi 64 bit hosts. -* `make debug arm32` builds only debug for gnu/linux arm 32 bit hosts. `./configure` -`./configure --host=x86_64-w64-mingw32 --with-zlib=3rdparty/zlib-1.2.11-winapi --with-openssl=3rdparty/openssl-1.0.2-winapi` - -`./configure --host=arm-linux-gnueabihf --with-fuse=3rdparty/libfuse-arm/usr --with-openssl=3rdparty/openssl-1.0.2-arm --with-zlib=3rdparty/zlib-1.2.11-arm` +`./configure --host=x86_64-w64-mingw32 --disable-media --with-zlib=3rdparty/zlib-1.3-winapi --with-openssl=3rdparty/openssl-1.0.2-winapi --with-librsync=3rdparty/librsync-winapi` ### Compiling on Darwin diff --git a/configure b/configure index 65c033c..55366e5 100755 --- a/configure +++ b/configure @@ -4594,7 +4594,7 @@ printf "%s\n" "found" >&6; } else with_openssl=$(realpath ${with_openssl}) OPENSSL_CFLAGS="-I${with_openssl}/include -DOPENSSL_API_COMPAT=0x10100000L" - OPENSSL_LIBS="${with_openssl}/lib/libcrypto.a" + OPENSSL_LIBS="${with_openssl}/libcrypto.a" if ! test -r ${with_openssl}/include/openssl/sha.h then as_fn_error $? "Expected openssl/sha.h to be find!" "$LINENO" 5 @@ -4721,6 +4721,8 @@ fi printf "%s\n" "found" >&6; } fi +else + echo "media disabled" fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sizeof time_t" >&5 diff --git a/configure.ac b/configure.ac index b1b771c..a4b702e 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -# Copyright (C) 2017 Fredrik Öhrström +# Copyright (C) 2017-2023 Fredrik Öhrström # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -191,7 +191,7 @@ if test "x${with_openssl}" = "x"; then else with_openssl=$(realpath ${with_openssl}) OPENSSL_CFLAGS="-I${with_openssl}/include -DOPENSSL_API_COMPAT=0x10100000L" - OPENSSL_LIBS="${with_openssl}/lib/libcrypto.a" + OPENSSL_LIBS="${with_openssl}/libcrypto.a" if ! test -r ${with_openssl}/include/openssl/sha.h then AC_MSG_ERROR([Expected openssl/sha.h to be find!]) @@ -284,6 +284,8 @@ if test "${ENABLE_MEDIA}" = "yes"; then AC_MSG_RESULT([found]) fi +else + echo "media disabled" fi AC_MSG_CHECKING(for sizeof time_t) diff --git a/make/Main.mk b/make/Main.mk index 0c3fb26..d9f02f4 100644 --- a/make/Main.mk +++ b/make/Main.mk @@ -44,14 +44,31 @@ $(shell mkdir -p $(OUTPUT_ROOT)/$(TYPE)) VERBOSE?=@ +ifeq ($(PLATFORM),WINAPI) +STRIP_COMMAND:=true +endif + +MEDIA_SOURCES:=beak_importmedia.cc beak_servemedia.cc beak_indexmedia.cc media.cc +MEDIA_SOURCES:=$(addprefix $(SRC_ROOT)/src/,$(MEDIA_SOURCES)) +NO_MEDIA_SOURCES:=no_beak_importmedia.cc no_beak_servemedia.cc no_beak_indexmedia.cc no_media.cc +NO_MEDIA_SOURCES:=$(addprefix $(SRC_ROOT)/src/,$(NO_MEDIA_SOURCES)) + WINAPI_SOURCES:=$(filter-out %posix.cc, $(wildcard $(SRC_ROOT)/src/*.cc)) ifeq ($(ENABLE_FUSE),yes) WINAPI_SOURCES:=$(filter-out %no_fuse.cc,$(WINAPI_SOURCES)) endif +WINAPI_SOURCES:=$(filter-out %media.cc,$(WINAPI_SOURCES)) + WINAPI_OBJS:=\ $(patsubst %.cc,%.o,$(subst $(SRC_ROOT)/src,$(OUTPUT_ROOT)/$(TYPE),$(WINAPI_SOURCES))) +WINAPI_MEDIA_OBJS:=\ + $(patsubst %.cc,%.o,$(subst $(SRC_ROOT)/src,$(OUTPUT_ROOT)/$(TYPE),$(MEDIA_SOURCES))) + +WINAPI_NO_MEDIA_OBJS:=\ + $(patsubst %.cc,%.o,$(subst $(SRC_ROOT)/src,$(OUTPUT_ROOT)/$(TYPE),$(NO_MEDIA_SOURCES))) + WINAPI_BEAK_OBJS:=\ $(filter-out %testinternals.o,$(WINAPI_OBJS)) @@ -60,6 +77,7 @@ $(OUTPUT_ROOT)/$(TYPE)/libgcc_s_seh-1.dll \ $(OUTPUT_ROOT)/$(TYPE)/libstdc++-6.dll \ $(OUTPUT_ROOT)/$(TYPE)/libwinpthread-1.dll + WINAPI_TESTINTERNALS_OBJS:=\ $(filter-out %main.o,$(WINAPI_OBJS)) @@ -71,11 +89,6 @@ endif POSIX_SOURCES:=$(filter-out %media.cc,$(POSIX_SOURCES)) -MEDIA_SOURCES:=beak_importmedia.cc beak_servemedia.cc beak_indexmedia.cc media.cc -MEDIA_SOURCES:=$(addprefix $(SRC_ROOT)/src/,$(MEDIA_SOURCES)) -NO_MEDIA_SOURCES:=no_beak_importmedia.cc no_beak_servemedia.cc no_beak_indexmedia.cc no_media.cc -NO_MEDIA_SOURCES:=$(addprefix $(SRC_ROOT)/src/,$(NO_MEDIA_SOURCES)) - POSIX_OBJS:=\ $(patsubst %.cc,%.o,$(subst $(SRC_ROOT)/src,$(OUTPUT_ROOT)/$(TYPE),$(POSIX_SOURCES))) @@ -115,8 +128,8 @@ $(OUTPUT_ROOT)/$(TYPE)/fileinfo.o: $(OUTPUT_ROOT)/generated_filetypes.h $(OUTPUT_ROOT)/$(TYPE)/%.o: $(SRC_ROOT)/src/%.cc @echo Compiling $(TYPE) $(CONF_MNEMONIC) $$(basename $<) - $(VERBOSE)$(CXX) $(CXXFLAGS_$(TYPE)) $(CXXFLAGS) -I$(OUTPUT_ROOT) -I$(BUILD_ROOT) -MMD $< -c -o $@ - $(VERBOSE)$(CXX) -E $(CXXFLAGS_$(TYPE)) $(CXXFLAGS) -I$(OUTPUT_ROOT) -I$(BUILD_ROOT) -MMD $< -c > $@.source + $(VERBOSE)$(CXX) -I$(OUTPUT_ROOT) -I$(BUILD_ROOT) $(CXXFLAGS_$(TYPE)) $(CXXFLAGS) -MMD $< -c -o $@ + $(VERBOSE)$(CXX) -E -I$(OUTPUT_ROOT) -I$(BUILD_ROOT) $(CXXFLAGS_$(TYPE)) $(CXXFLAGS) -MMD $< -c > $@.source $(OUTPUT_ROOT)/$(TYPE)/beak-media: $(BEAK_OBJS) $(BEAK_MEDIA_OBJS) @echo Linking $(TYPE) $(CONF_MNEMONIC) $@ @@ -125,6 +138,7 @@ $(OUTPUT_ROOT)/$(TYPE)/beak-media: $(BEAK_OBJS) $(BEAK_MEDIA_OBJS) $(VERBOSE)$(STRIP_COMMAND) $@ @echo Done linking $(TYPE) $(CONF_MNEMONIC) $@ +$(info BEAK_NO_MEDIA_OBJS=$(BEAK_NO_MEDIA_OBJS)) $(OUTPUT_ROOT)/$(TYPE)/beak: $(BEAK_OBJS) $(BEAK_NO_MEDIA_OBJS) @echo Linking $(TYPE) $(CONF_MNEMONIC) $@ $(VERBOSE)$(CXX) -o $@ $(LDFLAGS_$(TYPE)) $(LDFLAGS) $(BEAK_OBJS) $(BEAK_NO_MEDIA_OBJS) \ @@ -139,11 +153,11 @@ $(OUTPUT_ROOT)/$(TYPE)/testinternals: $(TESTINTERNALS_OBJS) $(BEAK_NO_MEDIA_OBJS $(VERBOSE)$(STRIP_COMMAND) $@ @echo Done linking $(TYPE) $(CONF_MNEMONIC) $@ -$(OUTPUT_ROOT)/$(TYPE)/libgcc_s_seh-1.dll: /usr/lib/gcc/x86_64-w64-mingw32/5.3-win32/libgcc_s_seh-1.dll - cp /usr/lib/gcc/x86_64-w64-mingw32/5.3-win32/libgcc_s_seh-1.dll $@ +$(OUTPUT_ROOT)/$(TYPE)/libgcc_s_seh-1.dll: /usr/lib/gcc/x86_64-w64-mingw32/10-win32/libgcc_s_seh-1.dll + cp /usr/lib/gcc/x86_64-w64-mingw32/10-win32/libgcc_s_seh-1.dll $@ -$(OUTPUT_ROOT)/$(TYPE)/libstdc++-6.dll: /usr/lib/gcc/x86_64-w64-mingw32/5.3-win32/libstdc++-6.dll - cp /usr/lib/gcc/x86_64-w64-mingw32/5.3-win32/libstdc++-6.dll $@ +$(OUTPUT_ROOT)/$(TYPE)/libstdc++-6.dll: /usr/lib/gcc/x86_64-w64-mingw32/10-win32/libstdc++-6.dll + cp /usr/lib/gcc/x86_64-w64-mingw32/10-win32/libstdc++-6.dll $@ $(OUTPUT_ROOT)/$(TYPE)/libwinpthread-1.dll: /usr/x86_64-w64-mingw32/lib/libwinpthread-1.dll cp $< $@ diff --git a/src/filesystem_winapi.cc b/src/filesystem_winapi.cc index 7d73da7..72e4a02 100644 --- a/src/filesystem_winapi.cc +++ b/src/filesystem_winapi.cc @@ -120,6 +120,7 @@ struct FileSystemImplementationWinapi : FileSystem RC addWatch(Path *dir); int endWatch(); FILE *openAsFILE(Path *p, const char *mode); + void allowAccessTimeUpdates(); FileSystemImplementationWinapi() : FileSystem("FileSystemImplementationWinapi") {} @@ -496,3 +497,7 @@ FILE *FileSystemImplementationWinapi::openAsFILE(Path *p, const char *mode) { return fopen(p->c_str(), mode); } + +void FileSystemImplementationWinapi::allowAccessTimeUpdates() +{ +} diff --git a/src/main.cc b/src/main.cc index d7e2aa8..2316712 100644 --- a/src/main.cc +++ b/src/main.cc @@ -25,7 +25,6 @@ #include "storagetool.h" #include "system.h" -#include #include #include @@ -46,41 +45,19 @@ int main(int argc, char *argv[]) } } -int setStackSize() -{ - const rlim_t kStackSize = 32 * 1024 * 1024; // min stack size = 32 MB - struct rlimit rl; - int result; - - result = getrlimit(RLIMIT_STACK, &rl); - if (result == 0) - { - if (rl.rlim_cur < kStackSize) - { - rl.rlim_cur = kStackSize; - result = setrlimit(RLIMIT_STACK, &rl); - if (result != 0) - { - fprintf(stderr, "setrlimit returned result = %d\n", result); - } - } - } - - // ... - - return 0; -} int run(int argc, char *argv[]) { RC rc = RC::OK; - setStackSize(); - ::captureStartTime(); // First create the OS interface to invoke external commands like rclone and rsync. auto sys = newSystem(); + + // Increase the stack size. + sys->setStackSize(); + // Next create the interface to the local file system where we find: // the orgin files and directories, the beak configuration file, the rclone configuration file, // and the temporary/cache files. diff --git a/src/system.h b/src/system.h index 8bd848f..89c717d 100644 --- a/src/system.h +++ b/src/system.h @@ -63,6 +63,8 @@ struct System virtual RC umount(ptr fuse_mount) = 0; // The current user running the beak software. virtual std::string userName() = 0; + // Increase the stack size. + virtual void setStackSize() = 0; virtual ~System() = default; }; diff --git a/src/system_posix.cc b/src/system_posix.cc index 74b0ea6..b573091 100644 --- a/src/system_posix.cc +++ b/src/system_posix.cc @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef OSX64 #include #include @@ -229,6 +230,8 @@ struct SystemImplementation : System SystemImplementation(); ~SystemImplementation() = default; + void setStackSize(); + private: pid_t running_shell_pid_ {}; @@ -561,3 +564,26 @@ string SystemImplementation::userName() { return user_name_; } + +void SystemImplementation::setStackSize() +{ + const rlim_t kStackSize = 32 * 1024 * 1024; // min stack size = 32 MB + struct rlimit rl; + int result; + + result = getrlimit(RLIMIT_STACK, &rl); + if (result == 0) + { + if (rl.rlim_cur < kStackSize) + { + rl.rlim_cur = kStackSize; + result = setrlimit(RLIMIT_STACK, &rl); + if (result != 0) + { + fprintf(stderr, "setrlimit returned result = %d\n", result); + } + } + } + + // ... +} diff --git a/src/system_winapi.cc b/src/system_winapi.cc index 09cc7a3..7ac6ce9 100644 --- a/src/system_winapi.cc +++ b/src/system_winapi.cc @@ -40,6 +40,7 @@ struct SystemImplementationWinapi : System std::unique_ptr mount(Path *dir, FuseAPI *fuseapi, bool debug); RC umount(ptr fuse_mount); string userName(); + void setStackSize(); private: int *rooot {}; @@ -105,3 +106,7 @@ string SystemImplementationWinapi::userName() { return ""; } + +void SystemImplementationWinapi::setStackSize() +{ +} diff --git a/src/util.cc b/src/util.cc index df1d82d..7ddf6ee 100644 --- a/src/util.cc +++ b/src/util.cc @@ -886,9 +886,14 @@ string timeToString(uint64_t t) char buf[256]; memset(buf, 0, sizeof(buf)); time_t pp = t/(1000*1000*1000); +#ifdef PLATFORM_WINAPI + struct tm *tid = localtime(&pp); + strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", tid); +#else struct tm tid; localtime_r(&pp, &tid); strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tid); +#endif return buf; } @@ -896,9 +901,14 @@ string timeToString(time_t pp) { char buf[256]; memset(buf, 0, sizeof(buf)); +#ifdef PLATFORM_WINAPI + struct tm *tid = localtime(&pp); + strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", tid); +#else struct tm tid; localtime_r(&pp, &tid); strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tid); +#endif return buf; }