Merge pull request #67 from Matts966/feature/side-effect-first

Add side effect first option and doc

Makefile

@@ -1,38 +1,79 @@
 .PHONY: build-and-check
 build-and-check: test
-	make sample
+	make samples
 
 .PHONY: osx
 osx:
 	CC=g++ bazelisk build //alphasql:all
 	sudo cp ./bazel-bin/alphasql/alphadag /usr/local/bin
+	sudo chmod +x /usr/local/bin/alphadag
 	sudo cp ./bazel-bin/alphasql/alphacheck /usr/local/bin
+	sudo chmod +x /usr/local/bin/alphacheck
 
-.PHONY: sample
-sample: osx
-	ls -d samples/*/ | while read sample_path; do \
-		echo ""; \
-		alphadag --with_functions $$sample_path --output_path $$sample_path/with_funtcions/dag.dot \
-			--external_required_tables_output_path $$sample_path/with_funtcions/external_tables.txt \
-		 	> $$sample_path/with_funtcions/alphadag_stdout.txt 2> $$sample_path/with_funtcions/alphadag_stderr.txt; \
-		dot -Tpng $$sample_path/with_funtcions/dag.dot -o $$sample_path/with_funtcions/dag.png; \
-		alphadag --with_tables $$sample_path --output_path $$sample_path/with_tables/dag.dot \
-			--external_required_tables_output_path $$sample_path/with_tables/external_tables.txt \
-		 	> $$sample_path/with_tables/alphadag_stdout.txt 2> $$sample_path/with_tables/alphadag_stderr.txt; \
-		dot -Tpng $$sample_path/with_tables/dag.dot -o $$sample_path/with_tables/dag.png; \
-		alphadag --with_tables --with_functions $$sample_path --output_path $$sample_path/with_all/dag.dot \
-			--external_required_tables_output_path $$sample_path/with_all/external_tables.txt \
-		 	> $$sample_path/with_all/alphadag_stdout.txt 2> $$sample_path/with_all/alphadag_stderr.txt; \
-		dot -Tpng $$sample_path/with_all/dag.dot -o $$sample_path/with_all/dag.png; \
+.PHONY: samples
+samples: without_options with_functions with_tables with_all side_effect_first side_effect_first_with_tables
+
+.PHONY: without_options
+without_options: osx
+	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
 		alphadag $$sample_path --output_path $$sample_path/dag.dot \
 			--external_required_tables_output_path $$sample_path/external_tables.txt \
-		 	> $$sample_path/alphadag_stdout.txt 2> $$sample_path/alphadag_stderr.txt; \
+		  > $$sample_path/alphadag_stdout.txt 2> $$sample_path/alphadag_stderr.txt; \
 		dot -Tpng $$sample_path/dag.dot -o $$sample_path/dag.png; \
 		alphacheck $$sample_path/dag.dot \
 			--json_schema_path ./samples/sample-schema.json \
 			> $$sample_path/alphacheck_stdout.txt 2> $$sample_path/alphacheck_stderr.txt; \
 	done;
 
+.PHONY: with_functions
+with_functions: osx
+	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
+	  mkdir -p $$sample_path/with_functions; \
+		alphadag --with_functions $$sample_path --output_path $$sample_path/with_functions/dag.dot \
+      --external_required_tables_output_path $$sample_path/with_functions/external_tables.txt \
+      > $$sample_path/with_functions/alphadag_stdout.txt 2> $$sample_path/with_functions/alphadag_stderr.txt; \
+		dot -Tpng $$sample_path/with_functions/dag.dot -o $$sample_path/with_functions/dag.png; \
+	done;
+
+.PHONY: with_tables
+with_tables: osx
+	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
+	  mkdir -p $$sample_path/with_tables; \
+    alphadag --with_tables $$sample_path --output_path $$sample_path/with_tables/dag.dot \
+      --external_required_tables_output_path $$sample_path/with_tables/external_tables.txt \
+      > $$sample_path/with_tables/alphadag_stdout.txt 2> $$sample_path/with_tables/alphadag_stderr.txt; \
+    dot -Tpng $$sample_path/with_tables/dag.dot -o $$sample_path/with_tables/dag.png; \
+	done;
+
+.PHONY: with_all
+with_all: osx
+	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
+	  mkdir -p $$sample_path/with_all; \
+		alphadag --with_tables --with_functions $$sample_path --output_path $$sample_path/with_all/dag.dot \
+			--external_required_tables_output_path $$sample_path/with_all/external_tables.txt \
+		 	> $$sample_path/with_all/alphadag_stdout.txt 2> $$sample_path/with_all/alphadag_stderr.txt; \
+		dot -Tpng $$sample_path/with_all/dag.dot -o $$sample_path/with_all/dag.png; \
+	done;
+
+.PHONY: side_effect_first
+side_effect_first: osx
+	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
+	  mkdir -p $$sample_path/side_effect_first; \
+		alphadag --side_effect_first $$sample_path --output_path $$sample_path/side_effect_first/dag.dot \
+			--external_required_tables_output_path $$sample_path/side_effect_first/external_tables.txt \
+		 	> $$sample_path/side_effect_first/alphadag_stdout.txt 2> $$sample_path/side_effect_first/alphadag_stderr.txt; \
+    dot -Tpng $$sample_path/side_effect_first/dag.dot -o $$sample_path/side_effect_first/dag.png; \
+	done;
+
+side_effect_first_with_tables: osx
+	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
+	  mkdir -p $$sample_path/side_effect_first_with_tables; \
+	 	alphadag --side_effect_first --with_tables $$sample_path --output_path $$sample_path/side_effect_first_with_tables/dag.dot \
+			--external_required_tables_output_path $$sample_path/side_effect_first_with_tables/external_tables.txt \
+		 	> $$sample_path/side_effect_first_with_tables/alphadag_stdout.txt 2> $$sample_path/side_effect_first_with_tables/alphadag_stderr.txt; \
+		dot -Tpng $$sample_path/side_effect_first_with_tables/dag.dot -o $$sample_path/side_effect_first_with_tables/dag.png; \
+	done;
+
 .PHONY: test
 test: osx
 	CC=g++ bazelisk test //alphasql:all

README.md

@@ -16,6 +16,8 @@ You can quickly introduce AlphaSQL by [CI Example](#ci-example).
 - [Dependency Analysis](#extract-dag-from-sql-set)
     - Extract DAG from your SQL file set.
     - [Sample DAG output](#sample-dag-output)
+    - [Side effect first](#side-effect-first)
+    - [With tables and functions](#with-tables-and-functions)
 - [Parallel Execution](#parallel-execution)
     - Automatically parallelize your SQL file set.
     - Integrate with Airflow.
@@ -86,9 +88,21 @@ The image below is extracted from SQL set in [./samples/sample](./samples/sample
 
 ![dag.dot](samples/sample/dag.png)
 
+### Side effect first
+
+You can resolve side effects such as `INSERT` and `UPDATE` statements before simple references by the `--side_effect_first` option.
+
+![dag.dot](./samples/sample-undefined/side_effect_first/dag.png)
+
+### With tables and functions
+
+You can extract dependencies containing tables and functions by `--with_tables` and `--with_functions` options.
+
+![dag.dot](./samples/sample/with_all/dag.png)
+
 ## Parallel Execution
 
-For BigQuery, the output DAG can be run parallely using 
+For BigQuery, the output DAG can be run parallely using
 
 - [bq-airflow-dag-generator](https://pypi.org/project/bq-airflow-dag-generator)
 - [bq-jobrunner](https://github.com/tsintermax/bq_jobrunner)
@@ -230,7 +244,7 @@ You can try the example CI with `gcloud` command by
 (cd ./samples/sample-ci && gcloud builds submit --config=cloudbuild_ci_sample.yaml .)
 ```
 
-This example 
+This example
 
 - Supports `_TABLE_SUFFIX` feature!
 - Does not execute actual BigQuery and very cheap!