Added t8117, t8118, t8119 - support for variants in subdirectories.

Variants were originally limited to be subdirectories of the top-level
directory (ie: siblings of the .tup directory itself). This was largely
to simplify the already complex problem of creating variants in the
first place, but now it appears to be an artificial limitation.

Fixing this mostly involved removing the safeguards that prevented
creating variants in subdirectories, as well as better support for
printing multi-part variant directories in print_tup_entry().

You still can't create a variant inside another variant, which is what
t8118 and t8119 check for. But you can create a single "builds"
directory and create all variants inside of that, for example.

src/tup/create_name_file.c

@@ -199,17 +199,12 @@ tupid_t tup_file_mod_mtime(tupid_t dt, const char *file, struct timespec mtime,
 	if(new || changed) {
 		if(modified) *modified = 1;
 		if(strcmp(file, TUP_CONFIG) == 0) {
-			/* tup.config only counts if it's at the project root, or if
-			 * it's in a top-level subdirectory for a variant.
+			/* If tup.config was modified, put the node in the
+			 * config list so we can import any variables that
+			 * have changed.
 			 */
-			if(tent->dt == DOT_DT || tent->parent->dt == DOT_DT) {
-				/* If tup.config was modified, put the node in
-				 * the config list so we can import any
-				 * variables that have changed.
-				 */
-				if(tup_db_add_config_list(tent->tnode.tupid) < 0)
-					return -1;
-			}
+			if(tup_db_add_config_list(tent->tnode.tupid) < 0)
+				return -1;
 		}
 	}
 

src/tup/db.c

@@ -4889,14 +4889,9 @@ static int save_vardict_file(struct vardb *vdb, const char *vardict_file)
 	if(tup_db_var_changed == 0)
 		return 0;
 
-	if(chdir(get_tup_top()) < 0) {
-		perror(get_tup_top());
-		fprintf(stderr, "tup error: Unable to change directory to project root.\n");
-		return -1;
-	}
-	fd = open(vardict_file, O_CREAT|O_WRONLY|O_TRUNC, 0666);
+	fd = openat(tup_top_fd(), vardict_file, O_CREAT|O_WRONLY|O_TRUNC, 0666);
 	if(fd < 0) {
-		perror("openat");
+		perror(vardict_file);
 		fprintf(stderr, "tup error: Unable to create the vardict file: '%s'\n", vardict_file);
 		return -1;
 	}

src/tup/entry.c

@@ -195,16 +195,14 @@ void tup_entry_set_verbose(int verbose)
 /* Returns 0 in case if a root tup entry has been passed and thus nothing has
  * been printed, otherwise 1 is returned.
  */
-static int print_tup_entry_internal(FILE *f, struct tup_entry *tent)
+static int print_tup_entry_internal(FILE *f, struct tup_entry *tent, struct tup_entry *stop_at_tent)
 {
-	/* Skip empty entries, and skip '.' here (tent->parent == NULL) */
-	if(!tent || !tent->parent)
-		return 0;
-	if(tup_entry_variant_null(tent) != tup_entry_variant_null(tent->parent)) {
-		fprintf(f, "[%s] ", tent->name.s);
+	/* Skip empty entries, and skip '.' here (tent->parent == NULL), and
+	 * stop if we get to the start of the variant part (stop_at_tent)
+	 */
+	if(!tent || !tent->parent || tent == stop_at_tent)
 		return 0;
-	}
-	if(print_tup_entry_internal(f, tent->parent))
+	if(print_tup_entry_internal(f, tent->parent, stop_at_tent))
 		fprintf(f, "%c", path_sep());
 	/* Don't print anything for the slash root entry */
 	if(tent->name.s[0] != '/')
@@ -216,23 +214,45 @@ void print_tup_entry(FILE *f, struct tup_entry *tent)
 {
 	const char *name;
 	int name_sz = 0;
+	struct variant *variant;
+	struct tup_entry *variant_part = NULL;
 
 	if(!tent)
 		return;
-	if(print_tup_entry_internal(f, tent->parent)) {
-		if(tent->type == TUP_NODE_CMD) {
-			fprintf(f, ": ");
-		} else {
-			fprintf(f, "%c", path_sep());
+
+	/* If we're in a variant, first print the variant part of the path in
+	 * brackets, like: [build/debug]
+	 */
+	variant = tup_entry_variant_null(tent);
+	if(tent->parent && variant && !variant->root_variant) {
+		fprintf(f, "[");
+		variant_part = tent;
+		while(variant_part && variant_part->parent && tup_entry_variant_null(variant_part->parent) == variant) {
+			variant_part = variant_part->parent;
 		}
+		print_tup_entry_internal(f, variant_part, NULL);
+		fprintf(f, "] ");
 	}
-	if(tent->parent && tup_entry_variant_null(tent) != tup_entry_variant_null(tent->parent)) {
-		fprintf(f, "[%s] ", tent->name.s);
-		name = ".";
-		name_sz = 1;
-	} else {
+
+	/* Now print the rest of the path */
+	if(tent != variant_part) {
+		if(print_tup_entry_internal(f, tent->parent, variant_part)) {
+			if(tent->type == TUP_NODE_CMD) {
+				fprintf(f, ": ");
+			} else {
+				fprintf(f, "%c", path_sep());
+			}
+		}
 		name = tent->name.s;
 		name_sz = tent->name.len;
+	} else {
+		/* If we're printing the variant directory itself (like the
+		 * path to build-debug or builds/debug, then print just a "."
+		 * since the whole path itself will be included in the variant
+		 * brackets earlier.
+		 */
+		name = ".";
+		name_sz = 1;
 	}
 	if(!do_verbose && tent->display) {
 		name = tent->display;

src/tup/updater.c

@@ -865,13 +865,6 @@ static int is_valid_variant_tent(struct tup_entry *tent)
 	 */
 	if(tent->type == TUP_NODE_GHOST)
 		return 0;
-
-	/* If the variant directory was moved with the monitor running, the
-	 * build directory no longer has DOT_DT as a parent. We can also remove
-	 * the variant in this case (t8063).
-	 */
-	if(tent->parent && tent->parent->dt != DOT_DT)
-		return 0;
 	return 1;
 }
 

src/tup/variant.c

@@ -94,7 +94,16 @@ int variant_add(struct tup_entry *tent, int enabled, struct variant **dest)
 		variant->vardict_len = snprintf(variant->vardict_file, sizeof(variant->vardict_file), ".tup/vardict") + 1;
 	} else {
 		variant->root_variant = 0;
-		variant->vardict_len = snprintf(variant->vardict_file, sizeof(variant->vardict_file), ".tup/vardict-%s", variant->variant_dir+1) + 1;
+
+		/* The vardict naming here could all be vardict-%lli, but for
+		 * backwards compatbility, top-level variants
+		 * (tent->parent->dt == DOT_DT) use the name of the variant.
+		 */
+		if(tent->parent->dt == DOT_DT) {
+			variant->vardict_len = snprintf(variant->vardict_file, sizeof(variant->vardict_file), ".tup/vardict-%s", variant->variant_dir+1) + 1;
+		} else {
+			variant->vardict_len = snprintf(variant->vardict_file, sizeof(variant->vardict_file), ".tup/vardict-%lli", variant->dtnode.tupid) + 1;
+		}
 	}
 	if(variant->vardict_len >= (signed)sizeof(variant->vardict_file)) {
 		fprintf(stderr, "tup error: variant vardict_file is sized incorrectly.\n");

test/t8011-variant-extra-tupconfig.sh

@@ -16,7 +16,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-# Make sure a tup.config not at a top-level doesn't create a new variant.
+# Make sure a tup.config not at the top-level creates a new variant.
 . ./tup.sh
 
 mkdir build
@@ -36,6 +36,9 @@ HERE
 touch sub/foo.c
 update
 
-check_not_exist build2/debug/sub
+check_not_exist build/sub/foo
+check_exist build/sub/bar
+check_exist build2/debug/sub/foo
+check_exist build2/debug/sub/bar
 
 eotup

test/t8062-move-variant-dir.sh

@@ -34,13 +34,18 @@ update
 tup_object_exist build-foo/tup.config FOO
 
 mv build-foo sub
+update_fail_msg "tup error: Please clean out the variant directory of extra files"
+
+# Now remove the extra files from the moved variant so it can be rebuilt.
+rm -rf sub/build-foo/configs
+rm -rf sub/build-foo/sub
 update
 
-# When we move the variant directory and detect with the scanner, all of the
-# generated nodes become normal nodes.
-check_exist sub/foo.o sub/bar.o
+# Now we get files built in the new variant.
 check_exist sub/build-foo/sub/foo.o sub/build-foo/sub/bar.o
 
+# The new tup.config is actually an invalid symlink since it was created with
+# 'tup variant', so we can't read any config variables from it.
 tup_object_no_exist sub/build-foo/tup.config FOO
 
 eotup

test/t8063-move-variant-dir-monitor.sh

@@ -40,10 +40,11 @@ mv build-foo sub
 update
 
 # When we move the variant directory and detect with the monitor, all of the
-# generated nodes become normal nodes.
-check_exist sub/foo.o sub/bar.o
+# files simply get rebuilt.
 check_exist sub/build-foo/sub/foo.o sub/build-foo/sub/bar.o
 
+# Even though the variant is valid, there are no variables because tup.config
+# is now an invalid symlink.
 tup_object_no_exist sub/build-foo/tup.config FOO
 
 stop_monitor

test/t8117-variant-in-subdir.sh

@@ -0,0 +1,47 @@
+#! /bin/sh -e
+# tup - A file-based build system
+#
+# Copyright (C) 2012-2024  Mike Shal <marfey@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Try a variant in a subdir not in a parent of the main dir.
+. ./tup.sh
+
+mkdir builds
+mkdir builds/variant1
+mkdir builds/variant2
+mkdir sub
+
+cat > Tupfile << HERE
+srcs += baz.c
+ifeq (@(FOO),y)
+srcs += foo.c
+endif
+: foreach \$(srcs) |> gcc -c %f -o %o |> %B.o
+HERE
+cat > sub/Tupfile << HERE
+: foreach bar.c |> gcc -c %f -o %o |> %B.o
+HERE
+echo "int main(void) {return 0;}" > foo.c
+echo "CONFIG_FOO=y" > builds/variant1/tup.config
+echo "CONFIG_FOO=n" > builds/variant2/tup.config
+touch sub/bar.c baz.c foo.c
+update
+
+check_exist builds/variant1/foo.o builds/variant1/baz.o builds/variant1/sub/bar.o
+check_exist builds/variant2/baz.o builds/variant2/sub/bar.o
+check_not_exist builds/variant2/foo.o
+
+eotup

test/t8118-variant-in-variant.sh

@@ -0,0 +1,47 @@
+#! /bin/sh -e
+# tup - A file-based build system
+#
+# Copyright (C) 2012-2024  Mike Shal <marfey@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Try a variant inside another variant.
+. ./tup.sh
+
+mkdir build
+mkdir sub
+
+cat > Tupfile << HERE
+srcs += baz.c
+ifeq (@(FOO),y)
+srcs += foo.c
+endif
+: foreach \$(srcs) |> gcc -c %f -o %o |> %B.o
+HERE
+cat > sub/Tupfile << HERE
+: foreach bar.c |> gcc -c %f -o %o |> %B.o
+HERE
+echo "int main(void) {return 0;}" > foo.c
+echo "CONFIG_FOO=y" > build/tup.config
+touch sub/bar.c baz.c foo.c
+update
+
+check_exist build/foo.o build/baz.o build/sub/bar.o
+
+mkdir build/build2
+touch build/build2/tup.config
+
+update_fail_msg "Unable to clean up old directory in the build tree:"
+
+eotup

test/t8119-variant-in-variant2.sh

@@ -0,0 +1,44 @@
+#! /bin/sh -e
+# tup - A file-based build system
+#
+# Copyright (C) 2012-2024  Mike Shal <marfey@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Like t8118, but this time both the variant and the sub variant are created at
+# the same time. This goes to a slightly different error path.
+. ./tup.sh
+
+mkdir build
+mkdir build/build2
+mkdir sub
+
+cat > Tupfile << HERE
+srcs += baz.c
+ifeq (@(FOO),y)
+srcs += foo.c
+endif
+: foreach \$(srcs) |> gcc -c %f -o %o |> %B.o
+HERE
+cat > sub/Tupfile << HERE
+: foreach bar.c |> gcc -c %f -o %o |> %B.o
+HERE
+echo "int main(void) {return 0;}" > foo.c
+echo "CONFIG_FOO=y" > build/tup.config
+touch sub/bar.c baz.c foo.c
+touch build/build2/tup.config
+
+update_fail_msg "Variant directory must only contain a tup.config file. Found extra files:"
+
+eotup

tup.1

@@ -1,4 +1,4 @@
-.TH "tup" "1" "2023/10/14" "http://gittup.org/tup" "tup manual"
+.TH "tup" "1" "2024/05/18" "http://gittup.org/tup" "tup manual"
 .\" disable hyphenation/justification
 .nh
 .ad l
@@ -860,7 +860,7 @@ TUP_PLATFORM is a special @-variable. If CONFIG_TUP_PLATFORM is not set in the t
 TUP_ARCH is another special @-variable. If CONFIG_TUP_ARCH is not set in the tup.config file, it has a default value according to the processor architecture that tup itself was compiled in. Currently the default value is one of "i386", "x86_64", "powerpc", "powerpc64", "ia64", "alpha", "sparc", "riscv32", "riscv64", "arm64", "arm" or "s390x".
 
 .SH "VARIANTS"
-Tup supports variants, which allow you to build your project multiple times with different configurations. Perhaps the most common case is to build a release and a debug configuration with different compiler flags, though any number of variants can be used to support whatever configurations you like. Each variant is built in its own directory distinct from each other and from the source tree. When building with variants, the in-tree build is disabled. To create a variant, make a new directory at the top of the tup hierarchy and create a "tup.config" file there. For example:
+Tup supports variants, which allow you to build your project multiple times with different configurations. Perhaps the most common case is to build a release and a debug configuration with different compiler flags, though any number of variants can be used to support whatever configurations you like. Each variant is built in its own directory distinct from each other and from the source tree. When building with variants, the in-tree build is disabled. To create a variant, make a new directory and create a "tup.config" file there. For example:
 .nf
 
 $ mkdir build-default
@@ -869,7 +869,7 @@ $ tup
 
 .fi
 
-Here we created a directory called "build-default" and made an empty tup.config inside. Note that the build directory must be at the same level as the ".tup" directory. Upon updating, tup will parse all of the Tupfiles using the configuration file we created, and place all build products within subdirectories of build-default that mirror the source tree. We could then create another variant like so:
+Here we created a directory called "build-default" and made an empty tup.config inside. Upon updating, tup will parse all of the Tupfiles using the configuration file we created, and place all build products within subdirectories of build-default that mirror the source tree. We could then create another variant like so:
 
 .nf
 
@@ -905,7 +905,7 @@ When using in-tree builds, the resulting build outputs may rely on run-time file
 
 .nf
 
-:foreach *.png |> !tup_preserve |>
+: foreach *.png |> !tup_preserve |>
 
 .fi