diff --git a/Advanced/CsvStreaming/Readme.md b/Advanced/CsvStreaming/Readme.md deleted file mode 100644 index ef12dd31..00000000 --- a/Advanced/CsvStreaming/Readme.md +++ /dev/null @@ -1,7 +0,0 @@ -## Streaming of large CSV documents - -Streaming in Templater is done by multiple calls to process API. -This allows to Templater to flush the content of populated stream and reuse memory in next call to process API. - -Streaming can be done only up to row without tags. This means that first non-streaming tags should be processed (if there are any) -and then streaming tags can be processed which will perform flushing. \ No newline at end of file diff --git a/Advanced/README.md b/Advanced/README.md index 2b392b6c..0cd69b5c 100644 --- a/Advanced/README.md +++ b/Advanced/README.md @@ -44,11 +44,12 @@ Consuming embedded CSV or Excel table via Power Query (Requires Excel 2010+) [template](PowerQuery/template/PowerQuery.xlsx?raw=true) - [result](PowerQuery/result.xlsx?raw=true) -### [CSV streaming](CsvStreaming/Readme.md) +### [CSV streaming](Streaming/Readme.md) -Stream CSV while processing to support huge exports +Stream CSV/XML while processing to support huge exports -[template](CsvStreaming/template/input.csv) - [result](CsvStreaming/result.csv) +[csv template](Streaming/template/input.csv) - [result](Streaming/result.csv) +[xml template](Streaming/template/input.xml) - [result](Streaming/result.xml) ### [Various JSON examples](TemplaterServer/Readme.md) diff --git a/Advanced/Streaming/Readme.md b/Advanced/Streaming/Readme.md new file mode 100644 index 00000000..cf1f4946 --- /dev/null +++ b/Advanced/Streaming/Readme.md @@ -0,0 +1,9 @@ +## Streaming of large documents + +Streaming in Templater is supported out of the box if streaming type is used (ResultSet/Iterator/Enumerator). +Alternatively streaming can be simulated manually by multiple calls to process API. + +Both methods allows Templater to flush the content of populated stream and reuse memory in next call to process API. + +Streaming can be done only up to row without tags. This means that first non-streaming tags should be processed (if there are any) +and then streaming tags can be processed which will perform flushing. \ No newline at end of file diff --git a/Advanced/CsvStreaming/CsvStreaming.csproj b/Advanced/Streaming/Streaming.csproj similarity index 91% rename from Advanced/CsvStreaming/CsvStreaming.csproj rename to Advanced/Streaming/Streaming.csproj index 305735c7..1920dd92 100644 --- a/Advanced/CsvStreaming/CsvStreaming.csproj +++ b/Advanced/Streaming/Streaming.csproj @@ -8,8 +8,8 @@ {5BB2AABB-A28F-404F-8C37-DBE122E893F5} Exe Properties - CsvStreaming - CsvStreaming + Streaming + Streaming v4.0 Client 512 @@ -39,7 +39,7 @@ ..\..\packages\DotNetZip.1.13.0\lib\net40\DotNetZip.dll True - + ..\..\packages\Templater.7.0.0\lib\Net40\NGS.Templater.dll False @@ -65,6 +65,11 @@ Always + + + Always + + diff --git a/Advanced/CsvStreaming/packages.config b/Advanced/Streaming/packages.config similarity index 100% rename from Advanced/CsvStreaming/packages.config rename to Advanced/Streaming/packages.config diff --git a/Advanced/CsvStreaming/pom.xml b/Advanced/Streaming/pom.xml similarity index 94% rename from Advanced/CsvStreaming/pom.xml rename to Advanced/Streaming/pom.xml index f86e46f6..a980f2b1 100644 --- a/Advanced/CsvStreaming/pom.xml +++ b/Advanced/Streaming/pom.xml @@ -2,10 +2,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 hr.ngs.templater.example - csv-streaming-example + streaming-example jar 7.0.0 - CSV streaming + Streaming https://github.com/ngs-doo/TemplaterExamples diff --git a/Advanced/CsvStreaming/result.csv b/Advanced/Streaming/result.csv similarity index 100% rename from Advanced/CsvStreaming/result.csv rename to Advanced/Streaming/result.csv diff --git a/Advanced/Streaming/result.xml b/Advanced/Streaming/result.xml new file mode 100644 index 00000000..aaa9d0ef --- /dev/null +++ b/Advanced/Streaming/result.xml @@ -0,0 +1,165 @@ + + + + + 260 + 27. 07. 2019. + + reference0 + branch0 + + + + 260 + 27. 07. 2019. + + reference1 + branch1 + + + + 260 + 27. 07. 2019. + + reference2 + branch2 + + - + + 260 + 27. 07. 2019. + + reference3 + branch3 + + ... + + 261 + 27. 07. 2019. + + reference4 + branch4 + + IMPORTANT + + 261 + 27. 07. 2019. + + reference5 + branch5 + + REMINDER + + 261 + 27. 07. 2019. + + reference6 + branch6 + + something to look "into later + + 261 + 27. 07. 2019. + + reference7 + branch7 + + special" char, + + 261 + 27. 07. 2019. + + reference8 + branch8 + + + + 261 + 27. 07. 2019. + + reference9 + branch9 + + + + 261 + 27. 07. 2019. + + reference10 + branch10 + + - + + 261 + 27. 07. 2019. + + reference11 + branch11 + + ... + + 262 + 27. 07. 2019. + + reference12 + branch12 + + IMPORTANT + + 262 + 27. 07. 2019. + + reference13 + branch13 + + REMINDER + + 262 + 27. 07. 2019. + + reference14 + branch14 + + something to look "into later + + 262 + 27. 07. 2019. + + reference15 + branch15 + + special" char, + + 262 + 27. 07. 2019. + + reference16 + branch16 + + + + 262 + 27. 07. 2019. + + reference17 + branch17 + + + + 262 + 27. 07. 2019. + + reference18 + branch18 + + - + + 262 + 27. 07. 2019. + + reference19 + branch19 + + ... + + \ No newline at end of file diff --git a/Advanced/CsvStreaming/src/Program.cs b/Advanced/Streaming/src/Program.cs similarity index 50% rename from Advanced/CsvStreaming/src/Program.cs rename to Advanced/Streaming/src/Program.cs index fb5bcdad..284ff399 100644 --- a/Advanced/CsvStreaming/src/Program.cs +++ b/Advanced/Streaming/src/Program.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Data; using System.Diagnostics; @@ -8,7 +9,7 @@ using Ionic.Zip; using NGS.Templater; -namespace CsvStreaming +namespace Streaming { public class Program { @@ -45,7 +46,7 @@ struct StreamingRow public string verifiedBy; public DateTime verifiedOn; - public StreamingRow(DataTableReader reader) + public StreamingRow(IDataReader reader) { id = reader.GetInt32(0); amount = reader.GetDecimal(1); @@ -59,6 +60,22 @@ public StreamingRow(DataTableReader reader) verifiedBy = reader.IsDBNull(9) ? null : reader.GetString(9); verifiedOn = reader.GetDateTime(10); } + + public class ReaderIterator : IEnumerator + { + private readonly IDataReader Reader; + + public ReaderIterator(IDataReader reader) + { + this.Reader = reader; + } + + public StreamingRow Current { get { return new StreamingRow(Reader); } } + object IEnumerator.Current { get { return Current; } } + public bool MoveNext() { return Reader.Read(); } + public void Reset() { } + public void Dispose() { } + } } public static void Main(string[] args) @@ -97,42 +114,77 @@ public static void Main(string[] args) startTimestamp.AddMinutes(i) ); } - var reader = table.CreateDataReader(); - var config = Configuration.Builder.Include(Quoter); + var reader1 = table.CreateDataReader(); + var reader2 = table.CreateDataReader(); + var reader3 = table.CreateDataReader(); + var csvConfig = Configuration.Builder.Include(Quoter); //we need quoting as we are simulating CSV + var xmlConfig = Configuration.Builder; //we don't need quoting as XML is natively supported //if we are using a culture which has comma as decimal separator, change the output to dot //we could apply this always, but it adds a bit of overhead, so let's apply it conditionally if (Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Contains(",")) - config.Include(NumberAsDot); + { + csvConfig.Include(NumberAsDot); + xmlConfig.Include(NumberAsDot); + } + csvConfig.Streaming(50000);//by default streaming is 16k, lets leave the default for xml + var csvFactory = csvConfig.Build(); + var xmlFactory = xmlConfig.Build(); //for example purposes we will stream it a zip file using (var zip = new ZipOutputStream("output.zip")) { - zip.PutNextEntry("output.csv"); - using (var doc = config.Build().Open(File.OpenRead("template/input.csv"), "csv", zip)) + zip.PutNextEntry("manual.csv"); + var sw = Stopwatch.StartNew(); + ManualStreaming(reader1, csvFactory, zip); + Console.WriteLine("manual csv took: " + sw.ElapsedMilliseconds); + zip.PutNextEntry("automatic.csv"); + sw = Stopwatch.StartNew(); + AutomaticStreaming(reader2, csvFactory, "csv", zip); + Console.WriteLine("automatic csv took: " + sw.ElapsedMilliseconds); + zip.PutNextEntry("data.xml"); + sw = Stopwatch.StartNew(); + AutomaticStreaming(reader3, xmlFactory, "xml", zip); + Console.WriteLine("automatic xml took: " + sw.ElapsedMilliseconds); + } + Process.Start(new ProcessStartInfo("output.zip") { UseShellExecute = true }); + } + + private static void ManualStreaming(IDataReader reader, IDocumentFactory factory, ZipOutputStream zip) + { + using (var doc = factory.Open(File.OpenRead("template/input.csv"), "csv", zip)) + { + //streaming processing assumes we have only a single collection, which means we first need to process all other tags + doc.Process(new { filter = new { date = "All", user = "All" } }); + //to do a streaming processing we need to process collection in chunks + var chunk = new List(50000); + var hasData = reader.Read(); + while (hasData) { - //streaming processing assumes we have only a single collection, which means we first need to process all other tags - doc.Process(new { filter = new { date = "All", user = "All" } }); - //to do a streaming processing we need to process collection in chunks - var chunk = new List(50000); - var hasData = reader.Read(); - while (hasData) + //one way of doing streaming is first duplicating the template row (context) + doc.Templater.Resize(doc.Templater.Tags, 2); + //and then process that row with all known data + //this way we will have additional row to process (or remove) later + do { - //one way of doing streaming is first duplicating the template row (context) - doc.Templater.Resize(doc.Templater.Tags, 2); - //and then process that row with all known data - //this way we will have additional row to process (or remove) later - do - { - chunk.Add(new StreamingRow(reader)); - hasData = reader.Read(); - } while (chunk.Count < 50000 && hasData); - doc.Process(new { data = chunk }); - chunk.Clear(); - } - //remove remaining rows - doc.Templater.Resize(doc.Templater.Tags, 0); + chunk.Add(new StreamingRow(reader)); + hasData = reader.Read(); + } while (chunk.Count < 50000 && hasData); + doc.Process(new { data = chunk }); + chunk.Clear(); } + //remove remaining rows + doc.Templater.Resize(doc.Templater.Tags, 0); + } + } + + private static void AutomaticStreaming(IDataReader reader, IDocumentFactory factory, string extension, ZipOutputStream zip) + { + using (var doc = factory.Open(File.OpenRead("template/input." + extension), extension, zip)) + { + //we still want to make sure all non collection tags are processed first (or they are at the end of document) + doc.Process(new { filter = new { date = "All", user = "All" } }); + //for streaming lets just pass enumerator for processing + doc.Process(new { data = new StreamingRow.ReaderIterator(reader) }); } - Process.Start(new ProcessStartInfo("output.zip") { UseShellExecute = true }); } } } diff --git a/Advanced/CsvStreaming/src/main/java/hr/ngs/templater/example/CsvStreamingExample.java b/Advanced/Streaming/src/main/java/hr/ngs/templater/example/StreamingExample.java similarity index 51% rename from Advanced/CsvStreaming/src/main/java/hr/ngs/templater/example/CsvStreamingExample.java rename to Advanced/Streaming/src/main/java/hr/ngs/templater/example/StreamingExample.java index 8743a696..b9099d11 100644 --- a/Advanced/CsvStreaming/src/main/java/hr/ngs/templater/example/CsvStreamingExample.java +++ b/Advanced/Streaming/src/main/java/hr/ngs/templater/example/StreamingExample.java @@ -1,6 +1,7 @@ package hr.ngs.templater.example; import hr.ngs.templater.Configuration; +import hr.ngs.templater.DocumentFactory; import hr.ngs.templater.DocumentFactoryBuilder; import hr.ngs.templater.TemplateDocument; @@ -13,10 +14,11 @@ import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Iterator; import java.util.Locale; import java.util.zip.*; -public class CsvStreamingExample { +public class StreamingExample { static class Quoter implements DocumentFactoryBuilder.LowLevelReplacer { @@ -58,7 +60,7 @@ public static class StreamingRow { public String verifiedBy; public Timestamp verifiedOn; - public StreamingRow(ResultSet rs) throws SQLException { + public StreamingRow(ResultSet rs) throws SQLException { id = rs.getInt(1); amount = rs.getBigDecimal(2); date = rs.getDate(3); @@ -71,10 +73,35 @@ public StreamingRow(ResultSet rs) throws SQLException { verifiedBy = rs.getString(10); verifiedOn = rs.getTimestamp(11); } + + public static class RsIterator implements Iterator { + private final ResultSet rs; + private boolean hasNext; + + public RsIterator(ResultSet rs) throws SQLException { + this.rs = rs; + this.hasNext = rs.next(); + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public StreamingRow next() { + try { + StreamingRow row = new StreamingRow(rs); + hasNext = rs.next(); + return row; + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } } public static void main(final String[] args) throws Exception { - InputStream templateStream = CsvStreamingExample.class.getResourceAsStream("/input.csv"); File tmp = File.createTempFile("output", ".zip"); Class.forName("org.hsqldb.jdbcDriver"); @@ -114,41 +141,89 @@ public static void main(final String[] args) throws Exception { ins.setTimestamp(11, new java.sql.Timestamp(startTimestamp.plusMinutes(i / 1000).toInstant().toEpochMilli())); ins.execute(); } - ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM csv_data"); - DocumentFactoryBuilder config = Configuration.builder().include(new Quoter()); + ResultSet rs1 = conn.createStatement().executeQuery("SELECT * FROM csv_data"); + ResultSet rs2 = conn.createStatement().executeQuery("SELECT * FROM csv_data"); + ResultSet rs3 = conn.createStatement().executeQuery("SELECT * FROM csv_data"); + DocumentFactoryBuilder csvConfig = Configuration.builder().include(new Quoter()); + DocumentFactoryBuilder xmlConfig = Configuration.builder(); DecimalFormatSymbols dfs = new DecimalFormatSymbols(Locale.getDefault()); //if we are using a culture which has comma as decimal separator, change the output to dot //we could apply this always, but it adds a bit of overhead, so let's apply it conditionally if (dfs.getDecimalSeparator() == ',') { - config.include(new NumberAsComma()); + csvConfig.include(new NumberAsComma()); + xmlConfig.include(new NumberAsComma()); } + csvConfig.streaming(50000);//by default streaming is 16k, lets leave the default for xml + DocumentFactory csvFactory = csvConfig.build(); + DocumentFactory xmlFactory = xmlConfig.build(); //we can stream directly into a zipped stream/file ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(tmp)); - zos.putNextEntry(new ZipEntry("output.csv")); - TemplateDocument doc = config.build().open(templateStream, "csv", zos); - //streaming processing assumes we have only a single collection, which means we first need to process all other tags - doc.process(new Object() { public Object filter = new Object() { public String date = "All"; public String user = "All"; }; }); - //to do a streaming processing we need to process collection in chunks - ArrayList chunk = new ArrayList<>(50000); - boolean hasData = rs.next(); - while (hasData) { - //one way of doing streaming is first duplicating the template row (context) - doc.templater().resize(doc.templater().tags(), 2); - //and then process that row with all known data - //this way we will have additional row to process (or remove) later - do { - chunk.add(new StreamingRow(rs)); - hasData = rs.next(); - } while (chunk.size() < 50000 && hasData); - doc.process(new Object() { public ArrayList data = chunk; }); - chunk.clear(); - } - //remove remaining rows - doc.templater().resize(doc.templater().tags(), 0); - doc.close(); + zos.putNextEntry(new ZipEntry("manual.csv")); + long start = System.currentTimeMillis(); + manualStreaming(rs1, csvFactory, zos); + System.out.println("manual csv took: " + (System.currentTimeMillis() - start)); + zos.putNextEntry(new ZipEntry("automatic.csv")); + start = System.currentTimeMillis(); + automaticStreaming(rs2, csvFactory, "csv", zos); + System.out.println("automatic csv took: " + (System.currentTimeMillis() - start)); + zos.putNextEntry(new ZipEntry("data.xml")); + start = System.currentTimeMillis(); + //by default XML will do many small operations so its much faster to wrap the stream with a buffer + BufferedOutputStream bos = new BufferedOutputStream(zos); + automaticStreaming(rs3, xmlFactory, "xml", bos); + bos.flush(); + System.out.println("automatic xml took: " + (System.currentTimeMillis() - start)); conn.close(); - zos.closeEntry(); zos.close(); Desktop.getDesktop().open(tmp); } + + private static void manualStreaming(ResultSet rs, DocumentFactory factory, OutputStream os) throws SQLException { + InputStream templateStream = StreamingExample.class.getResourceAsStream("/input.csv"); + try (TemplateDocument doc = factory.open(templateStream, "csv", os)) { + //streaming processing assumes we have only a single collection, which means we first need to process all other tags + doc.process(new Object() { + public Object filter = new Object() { + public String date = "All"; + public String user = "All"; + }; + }); + //to do a streaming processing we need to process collection in chunks + ArrayList chunk = new ArrayList<>(50000); + boolean hasData = rs.next(); + while (hasData) { + //one way of doing streaming is first duplicating the template row (context) + doc.templater().resize(doc.templater().tags(), 2); + //and then process that row with all known data + //this way we will have additional row to process (or remove) later + do { + chunk.add(new StreamingRow(rs)); + hasData = rs.next(); + } while (chunk.size() < 50000 && hasData); + doc.process(new Object() { + public ArrayList data = chunk; + }); + chunk.clear(); + } + //remove remaining rows + doc.templater().resize(doc.templater().tags(), 0); + } + } + + private static void automaticStreaming(ResultSet rs, DocumentFactory factory, String extension, OutputStream os) throws SQLException { + InputStream templateStream = StreamingExample.class.getResourceAsStream("/input." + extension); + try (TemplateDocument doc = factory.open(templateStream, extension, os)) { + //we still want to make sure all non collection tags are processed first (or they are at the end of document) + doc.process(new Object() { + public Object filter = new Object() { + public String date = "All"; + public String user = "All"; + }; + }); + //for streaming lets just pass iterator for processing + doc.process(new Object() { + public Iterator data = new StreamingRow.RsIterator(rs); + }); + } + } } diff --git a/Advanced/CsvStreaming/template/input.csv b/Advanced/Streaming/template/input.csv similarity index 100% rename from Advanced/CsvStreaming/template/input.csv rename to Advanced/Streaming/template/input.csv diff --git a/Advanced/Streaming/template/input.xml b/Advanced/Streaming/template/input.xml new file mode 100644 index 00000000..d8d477b9 --- /dev/null +++ b/Advanced/Streaming/template/input.xml @@ -0,0 +1,15 @@ + + + + + + [[data.amount]] + [[data.date]:format] + + [[data.reference]] + [[data.branch]] + + [[data.note]] + + + diff --git a/Advanced/TemplaterServer/Dockerfile b/Advanced/TemplaterServer/Dockerfile index 5cabcfc9..8992961e 100644 --- a/Advanced/TemplaterServer/Dockerfile +++ b/Advanced/TemplaterServer/Dockerfile @@ -6,7 +6,7 @@ ENV TZ=Europe/Zagreb RUN apt update && apt install openjdk-11-jre-headless libreoffice-common libreoffice-java-common libreoffice-writer libreoffice-calc wget -yq -RUN wget -q https://github.com/ngs-doo/TemplaterExamples/releases/download/v6.1.0/templater-server.jar +RUN wget -q https://github.com/ngs-doo/TemplaterExamples/releases/download/v7.0.0/templater-server.jar COPY templater.lic . diff --git a/Beginner/AndroidExample/app/build.gradle b/Beginner/AndroidExample/app/build.gradle index 44483d38..4091b7cf 100644 --- a/Beginner/AndroidExample/app/build.gradle +++ b/Beginner/AndroidExample/app/build.gradle @@ -4,12 +4,16 @@ android { compileSdkVersion 28 defaultConfig { applicationId "hr.ngs.templater.example" - minSdkVersion 19 + minSdkVersion 26 targetSdkVersion 28 versionCode 1 versionName "1.0" testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } buildTypes { release { minifyEnabled false @@ -39,6 +43,7 @@ dependencies { implementation 'com.android.support.constraint:constraint-layout:1.1.3' implementation 'com.android.support:design:28.0.0' implementation 'hr.ngs.templater:templater:7.0.0' + implementation 'stax:stax:1.2.0' testImplementation 'junit:junit:4.12' androidTestImplementation 'com.android.support.test:runner:1.0.2' androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' diff --git a/Beginner/AndroidExample/app/src/main/java/hr/ngs/templater/example/Templater.java b/Beginner/AndroidExample/app/src/main/java/hr/ngs/templater/example/Templater.java index c9a9ef45..23478cdd 100644 --- a/Beginner/AndroidExample/app/src/main/java/hr/ngs/templater/example/Templater.java +++ b/Beginner/AndroidExample/app/src/main/java/hr/ngs/templater/example/Templater.java @@ -1,28 +1,24 @@ package hr.ngs.templater.example; -import org.apache.xerces.jaxp.DocumentBuilderFactoryImpl; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import hr.ngs.templater.Configuration; -import hr.ngs.templater.ITemplateDocument; +import hr.ngs.templater.TemplateDocument; public abstract class Templater { public static void createDocument(InputStream template, String extension, OutputStream result, Object ...data) throws IOException { //By default Templater will include Java images in low level plugins. //To avoid missing awt dependency disable low level plugins - //Use custom XML library as Android one does not work for non-trivial stuff - ITemplateDocument document = Configuration.builder() + TemplateDocument document = Configuration.builder() .builtInLowLevelPlugins(false) - .xmlBuilder(new org.apache.xerces.jaxp.DocumentBuilderFactoryImpl(), false) .build().open(template, extension, result); for(Object d : data) { document.process(d); } - document.flush(); + document.close(); template.close(); } } diff --git a/Beginner/AndroidExample/build.gradle b/Beginner/AndroidExample/build.gradle index 160cb824..bc090d89 100644 --- a/Beginner/AndroidExample/build.gradle +++ b/Beginner/AndroidExample/build.gradle @@ -6,7 +6,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.5.2' + classpath 'com.android.tools.build:gradle:4.0.2' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/Beginner/AndroidExample/gradle/wrapper/gradle-wrapper.properties b/Beginner/AndroidExample/gradle/wrapper/gradle-wrapper.properties index d3b65932..54fb3aa7 100644 --- a/Beginner/AndroidExample/gradle/wrapper/gradle-wrapper.properties +++ b/Beginner/AndroidExample/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Sun Dec 01 09:22:26 CET 2019 +#Tue Apr 19 15:52:59 CEST 2022 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-all.zip diff --git a/Beginner/DataSet (.NET)/Readme.md b/Beginner/DataSet (.NET)/Readme.md index bb556072..601f6072 100644 --- a/Beginner/DataSet (.NET)/Readme.md +++ b/Beginner/DataSet (.NET)/Readme.md @@ -57,4 +57,12 @@ In this case to specify background color for a cell, Word uses properties such a -As of v2.5 Templater can use merge-xml metadata as instruction to merge provided XML to the surrounding context. This way we can "append" color to the appropriate place. \ No newline at end of file +As of v2.5 Templater can use merge-xml metadata as instruction to merge provided XML to the surrounding context. This way we can "append" color to the appropriate place. + +As of v7 this merge-xml can be passed directly through XML so there is no need for it in tag metadata. In that case XML would look like: + + + + + + diff --git a/Intermediate/AlternativeProperty/src/Program.cs b/Intermediate/AlternativeProperty/src/Program.cs index 9bae44dc..e39875a9 100644 --- a/Intermediate/AlternativeProperty/src/Program.cs +++ b/Intermediate/AlternativeProperty/src/Program.cs @@ -21,18 +21,18 @@ class MyObject public MyObjectA objectA = new MyObjectA(); public MyObjectB objectB = new MyObjectB(); } - private static ThreadLocal currentRoot = new ThreadLocal(); public static void Main(string[] args) { File.Copy("template/Fields.docx", "Fields.docx", true); + object currentRoot = null; Func missingFormatter = (value, metadata) => { if (metadata.StartsWith("missing(") && value == null) { //path to appropriate field string[] path = metadata.Substring(8, metadata.Length - 9).Split('.'); - object current = currentRoot.Value; + object current = currentRoot; foreach (string p in path) { var f = current.GetType().GetField(p); @@ -44,20 +44,20 @@ public static void Main(string[] args) }; var factory = Configuration.Builder.Include(missingFormatter).Build(); using (var doc = factory.Open("Fields.docx")) - ProcessValue(doc, new MyObject()); + ProcessValue(ref currentRoot, doc, new MyObject()); Process.Start(new ProcessStartInfo("Fields.docx") { UseShellExecute = true }); } - private static void ProcessValue(ITemplateDocument doc, object value) + private static void ProcessValue(ref object currentRoot, ITemplateDocument doc, object value) { try { - currentRoot.Value = value; + currentRoot = value; doc.Process(value); } finally { - currentRoot.Value = null; + currentRoot = null; } } } diff --git a/Intermediate/AlternativeProperty/src/main/java/hr/ngs/templater/example/FieldsExample.java b/Intermediate/AlternativeProperty/src/main/java/hr/ngs/templater/example/FieldsExample.java index 90ae9e7f..2b2e4399 100644 --- a/Intermediate/AlternativeProperty/src/main/java/hr/ngs/templater/example/FieldsExample.java +++ b/Intermediate/AlternativeProperty/src/main/java/hr/ngs/templater/example/FieldsExample.java @@ -11,18 +11,22 @@ public class FieldsExample { static class MyObjectA { public String fieldA = null; } + static class MyObjectB { public String fieldB = "alternative value"; } + static class MyObject { public MyObjectA objectA = new MyObjectA(); public MyObjectB objectB = new MyObjectB(); } static class MissingFormatter implements DocumentFactoryBuilder.Formatter { - private Callable getRoot; - public MissingFormatter(Callable getRoot) { - this.getRoot = getRoot; + //to be able to navigate over non processed object, lets keep reference to entry point + private Object currentRootObject; + + public void setRoot(Object root) { + this.currentRootObject = root; } @Override @@ -31,8 +35,8 @@ public Object format(Object value, String metadata) { try { //path to appropriate field String[] path = metadata.substring(8, metadata.length() - 1).split("\\."); - Object current = getRoot.call(); - for(String p : path) { + Object current = currentRootObject; + for (String p : path) { Field f = current.getClass().getField(p); current = f.get(current); } @@ -44,31 +48,25 @@ public Object format(Object value, String metadata) { } } - private static final ThreadLocal currentRoot = new ThreadLocal(); - public static void main(final String[] args) throws Exception { InputStream templateStream = FieldsExample.class.getResourceAsStream("/Fields.docx"); File tmp = File.createTempFile("fields", ".docx"); FileOutputStream fos = new FileOutputStream(tmp); - DocumentFactory factory = Configuration.builder().include(new MissingFormatter(new Callable() { - @Override - public Object call() { - return currentRoot.get(); - } - })).build(); + MissingFormatter formatter = new MissingFormatter(); + DocumentFactory factory = Configuration.builder().include(formatter).build(); try (TemplateDocument tpl = factory.open(templateStream, "docx", fos)) { - process(tpl, new MyObject()); + process(formatter, tpl, new MyObject()); } fos.close(); Desktop.getDesktop().open(tmp); } - private static void process(TemplateDocument doc, Object value) { + private static void process(MissingFormatter formatter, TemplateDocument doc, Object value) { try { - currentRoot.set(value); + formatter.setRoot(value); // we can keep track of root object in a formatter plugin doc.process(value); } finally { - currentRoot.remove(); + formatter.setRoot(null); } } } diff --git a/Intermediate/HtmlToExcel/result.xlsx b/Intermediate/HtmlToExcel/result.xlsx index 6591cf82..0b362a93 100644 Binary files a/Intermediate/HtmlToExcel/result.xlsx and b/Intermediate/HtmlToExcel/result.xlsx differ diff --git a/Intermediate/HtmlToExcel/src/Program.cs b/Intermediate/HtmlToExcel/src/Program.cs index af7b8cd9..7d79e740 100644 --- a/Intermediate/HtmlToExcel/src/Program.cs +++ b/Intermediate/HtmlToExcel/src/Program.cs @@ -74,18 +74,42 @@ public static object ConverterHtml(object value, string metadata) return value; } + enum Color + { + RED = 0, + ORANGE = 1, + YELLOW = 2, + GREEN = 3, + BLUE = 4 + } + + static object ColorToXML(object value, string tag, string[] metadata) + { + if (value is Color) + { + var c = (Color)value; + //we need to know the location of conversion table in Excel (this could be provided as argument if needed) + var t = new XElement(XName.Get("t", "http://schemas.openxmlformats.org/spreadsheetml/2006/main")); + t.SetAttributeValue("templater-cell-style", "Colors!A" + (2 + (int)c)); + return t; + } + return value; + } + public static void Main(string[] args) { File.Copy("template/Document.xlsx", "Html.xlsx", true); var factory = Configuration.Builder .Include(ConverterHtml) + .Include(ColorToXML) .Build(); using (var doc = factory.Open("Html.xlsx")) { doc.Process(new { html = "

My simple bold text in red!

", - numbers = new[] { new Number(100), new Number(-100), new Number(10) } + numbers = new[] { new Number(100), new Number(-100), new Number(10) }, + background = Color.ORANGE }); } Process.Start(new ProcessStartInfo("Html.xlsx") { UseShellExecute = true }); diff --git a/Intermediate/HtmlToExcel/src/main/java/hr/ngs/templater/example/HtmlExcelExample.java b/Intermediate/HtmlToExcel/src/main/java/hr/ngs/templater/example/HtmlExcelExample.java index 7047a514..971b7e39 100644 --- a/Intermediate/HtmlToExcel/src/main/java/hr/ngs/templater/example/HtmlExcelExample.java +++ b/Intermediate/HtmlToExcel/src/main/java/hr/ngs/templater/example/HtmlExcelExample.java @@ -93,6 +93,41 @@ public Object format(Object value, String metadata) { } } + enum Color { + RED(0), + ORANGE(1), + YELLOW(2), + GREEN(3), + BLUE(4); + + public final int value; + + Color(int value) { + this.value = value; + } + } + + private static class ColorToXML implements DocumentFactoryBuilder.LowLevelReplacer { + DocumentBuilder dBuilder; + + ColorToXML(DocumentBuilder dBuilder) { + this.dBuilder = dBuilder; + } + + @Override + public Object replace(Object value, String tag, String[] metadata) { + if (value instanceof Color) { + Color c = (Color) value; + //we need to know the location of conversion table in Excel (this could be provided as argument if needed) + Element t = dBuilder.newDocument().createElement("t"); + t.setAttribute("templater-cell-style", "Colors!A" + (2 + c.value)); + return t; + } + return value; + } + } + + public static void main(final String[] args) throws Exception { DocumentBuilder dBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); @@ -102,9 +137,13 @@ public static void main(final String[] args) throws Exception { Map map = new HashMap(); map.put("html", "

My simple bold text in red!

"); map.put("numbers", Arrays.asList(new Number(100), new Number(-100), new Number(10))); + map.put("background", Color.ORANGE); try (FileOutputStream fos = new FileOutputStream(tmp); - TemplateDocument tpl = Configuration.builder().include(new HtmlToOoxml(dBuilder)).build().open(templateStream, "xlsx", fos)) { + TemplateDocument tpl = Configuration.builder() + .include(new HtmlToOoxml(dBuilder)) + .include(new ColorToXML(dBuilder)) + .build().open(templateStream, "xlsx", fos)) { tpl.process(map); } java.awt.Desktop.getDesktop().open(tmp); diff --git a/Intermediate/HtmlToExcel/template/Document.xlsx b/Intermediate/HtmlToExcel/template/Document.xlsx index b171a218..85f6be4c 100644 Binary files a/Intermediate/HtmlToExcel/template/Document.xlsx and b/Intermediate/HtmlToExcel/template/Document.xlsx differ diff --git a/Intermediate/HtmlToWord/Readme.md b/Intermediate/HtmlToWord/Readme.md index a873e33f..0cf05222 100644 --- a/Intermediate/HtmlToWord/Readme.md +++ b/Intermediate/HtmlToWord/Readme.md @@ -20,6 +20,8 @@ This will leave old paragraph empty, which might lead to whitespace bloat. To de * replace-xml - which will insert XML at the place of the paragraph (instead of after it) * merge-xml - which will merge provided XML into the found structure +To ease the usage, instead of specifying this through tag metadata, they can be sent in XML via templater-xml attribute, e.g. templater-xml="merge-xml" + ### Document merging With version 6.1 Templater also supports document embedding in Word. This can be used for document merging, HTML/RTF import and similar purposes. To import embedded document, special type must be used: diff --git a/Intermediate/HtmlToWord/src/Program.cs b/Intermediate/HtmlToWord/src/Program.cs index 854c6556..bcd1d892 100644 --- a/Intermediate/HtmlToWord/src/Program.cs +++ b/Intermediate/HtmlToWord/src/Program.cs @@ -35,7 +35,10 @@ public static object ComplexHtmlConverter(object value, string metadata) { var paragraphs = Converter.Parse(value.ToString()); //return collection of XElement objects which will be inserted as is into document current tag - return paragraphs.Select(it => XElement.Parse(it.OuterXml)); + var xmls = paragraphs.Select(it => XElement.Parse(it.OuterXml)).ToList(); + //lets put special attribute directly on XML so we don't need to put it on tag + xmls[0].SetAttributeValue("templater-xml", "remove-old-xml"); + return xmls; } return value; } diff --git a/Intermediate/HtmlToWord/src/main/java/hr/ngs/templater/example/HtmlWordExample.java b/Intermediate/HtmlToWord/src/main/java/hr/ngs/templater/example/HtmlWordExample.java index 15aab4f3..562bc4ab 100644 --- a/Intermediate/HtmlToWord/src/main/java/hr/ngs/templater/example/HtmlWordExample.java +++ b/Intermediate/HtmlToWord/src/main/java/hr/ngs/templater/example/HtmlWordExample.java @@ -63,10 +63,12 @@ public ComplexHtmlConverter(DocumentBuilder dBuilder) { public Object format(Object value, String metadata) { if (metadata.equals("complex-html")) { NodeList bodyNodes = convert(value.toString(), dBuilder).getChildNodes(); - List elements = new ArrayList(bodyNodes.getLength()); + List elements = new ArrayList<>(bodyNodes.getLength()); for (int i = 0; i < bodyNodes.getLength(); i++) { elements.add((Element) bodyNodes.item(i)); } + //lets put special attribute directly on XML so we don't need to put it on tag + elements.get(0).setAttribute("templater-xml", "remove-old-xml"); return elements.toArray(new Element[0]); } return value; diff --git a/Intermediate/HtmlToWord/template/template.docx b/Intermediate/HtmlToWord/template/template.docx index f3499431..03029028 100644 Binary files a/Intermediate/HtmlToWord/template/template.docx and b/Intermediate/HtmlToWord/template/template.docx differ diff --git a/Intermediate/README.md b/Intermediate/README.md index 2b1a1e0c..7950b787 100644 --- a/Intermediate/README.md +++ b/Intermediate/README.md @@ -4,7 +4,7 @@ Learning how to extend Templater via plugins and create more complicated reports ### [Combining basics](FoodOrder%20(.NET)/Readme.md) -The path to good looking reports is to create a good looking template. +The path to pretty reports is to create a pretty template. [Word template](FoodOrder%20(.NET)/FoodOrder.Web/App_Data/Order.docx?raw=true) - [result](FoodOrder%20(.NET)/result.docx?raw=true) diff --git a/README.md b/README.md index 15d50f50..eef1a459 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [documentation]: https://templater.info/ -# Reporting for JVM and .NET +# Document generation from templates Templater is a **reporting library** which can be **easily integrated into third party apps** as advanced document generation engine. It works by binding provided data with specified template. @@ -15,7 +15,7 @@ Additional [documentation] is available from the official website. ## How Templater works -Templater works by analyzing provided *docx/xlsx/pptx/csv* document for **tags**. +Templater works by analyzing provided *docx/xlsx/pptx/csv/xml* document for **tags**. Tags are snippets of text written in either **`[[tag]]`**, **`{{tag}}`** or a **`<>`** format. Tags can have metadata which can be used for various customization purposes, such as formatting, verbalization and others. @@ -67,6 +67,7 @@ It can work for simple cases, such as single row in a table, but can be used for * [pushdown/pushright](Beginner/PushDownExample) - elements bellow/right of tags area will be moved according to builtin rules * [merge cells/named ranges/tables](Beginner/NamedRange) - influence push rules by extending the affected region * [XML binding](Advanced/XmlBinding) - custom XML will be changed/updated when bound in Word + * [streaming](Advanced/Streaming) - streaming over collections is also supported ## Examples diff --git a/RunAll/Program.cs b/RunAll/Program.cs index e9ea8ddd..7f895a5e 100644 --- a/RunAll/Program.cs +++ b/RunAll/Program.cs @@ -46,7 +46,7 @@ static void Main(string[] args) //TemplaterWeb // web app. run manually DoubleProcessing.Program.Main(args); SheetReport.Program.Main(args); - CsvStreaming.Program.Main(args); + Streaming.Program.Main(args); XmlBinding.Program.Main(args); DepartmentReport.Program.Main(args); //PowerQuery.Program.Main(args);//requires license file to work properly diff --git a/RunAll/RunAll.csproj b/RunAll/RunAll.csproj index e7e8eae6..6f6a7a45 100644 --- a/RunAll/RunAll.csproj +++ b/RunAll/RunAll.csproj @@ -52,10 +52,6 @@ - - {5BB2AABB-A28F-404F-8C37-DBE122E893F5} - CsvStreaming - {B7765E9A-952F-46C7-9E2A-4DB7A1A045F6} DepartmentReport @@ -72,6 +68,10 @@ {B3457322-FDDB-46AC-B445-2887C27BC723} SheetReport + + {5BB2AABB-A28F-404F-8C37-DBE122E893F5} + Streaming + {716992A2-B22C-463B-9D33-78CDDFB8973E} XmlBinding diff --git a/Templater.sln b/Templater.sln index df7a6431..34edc580 100644 --- a/Templater.sln +++ b/Templater.sln @@ -97,8 +97,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PowerQuery", "Advanced\Powe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MissingProperty", "Intermediate\MissingProperty\MissingProperty.csproj", "{AF4F0BF6-87C2-4A49-AFF0-CA8FBE5C53F2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvStreaming", "Advanced\CsvStreaming\CsvStreaming.csproj", "{5BB2AABB-A28F-404F-8C37-DBE122E893F5}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimplePresentation", "Beginner\SimplePresentation\SimplePresentation.csproj", "{172BA03B-8863-4721-B22D-0DFD205CCF46}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedCharts", "Intermediate\SharedCharts\SharedCharts.csproj", "{25A57D11-D463-4293-817A-A967C2107CBF}" @@ -107,6 +105,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PresentationTables", "Begin EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Paragraphs", "Beginner\Paragraphs\Paragraphs.csproj", "{25A57D63-D463-4090-817B-B967C2107CBF}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Streaming", "Advanced\Streaming\Streaming.csproj", "{5BB2AABB-A28F-404F-8C37-DBE122E893F5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -567,16 +567,6 @@ Global {AF4F0BF6-87C2-4A49-AFF0-CA8FBE5C53F2}.Release|Mixed Platforms.Build.0 = Release|x86 {AF4F0BF6-87C2-4A49-AFF0-CA8FBE5C53F2}.Release|x86.ActiveCfg = Release|x86 {AF4F0BF6-87C2-4A49-AFF0-CA8FBE5C53F2}.Release|x86.Build.0 = Release|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|Any CPU.ActiveCfg = Debug|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|Mixed Platforms.ActiveCfg = Debug|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|Mixed Platforms.Build.0 = Debug|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|x86.ActiveCfg = Debug|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|x86.Build.0 = Debug|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|Any CPU.ActiveCfg = Release|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|Mixed Platforms.ActiveCfg = Release|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|Mixed Platforms.Build.0 = Release|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|x86.ActiveCfg = Release|x86 - {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|x86.Build.0 = Release|x86 {172BA03B-8863-4721-B22D-0DFD205CCF46}.Debug|Any CPU.ActiveCfg = Debug|x86 {172BA03B-8863-4721-B22D-0DFD205CCF46}.Debug|Mixed Platforms.ActiveCfg = Debug|x86 {172BA03B-8863-4721-B22D-0DFD205CCF46}.Debug|Mixed Platforms.Build.0 = Debug|x86 @@ -617,6 +607,16 @@ Global {25A57D63-D463-4090-817B-B967C2107CBF}.Release|Mixed Platforms.Build.0 = Release|x86 {25A57D63-D463-4090-817B-B967C2107CBF}.Release|x86.ActiveCfg = Release|x86 {25A57D63-D463-4090-817B-B967C2107CBF}.Release|x86.Build.0 = Release|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|Any CPU.ActiveCfg = Debug|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|Mixed Platforms.ActiveCfg = Debug|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|Mixed Platforms.Build.0 = Debug|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|x86.ActiveCfg = Debug|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Debug|x86.Build.0 = Debug|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|Any CPU.ActiveCfg = Release|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|Mixed Platforms.ActiveCfg = Release|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|Mixed Platforms.Build.0 = Release|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|x86.ActiveCfg = Release|x86 + {5BB2AABB-A28F-404F-8C37-DBE122E893F5}.Release|x86.Build.0 = Release|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/pom.xml b/pom.xml index 54d49135..a1d6e907 100644 --- a/pom.xml +++ b/pom.xml @@ -55,7 +55,7 @@ Intermediate/TemplaterJson Advanced/DoubleProcessing Advanced/SheetReport - Advanced/CsvStreaming + Advanced/Streaming Advanced/XmlBinding Advanced/PowerQuery Advanced/DepartmentReport @@ -271,7 +271,7 @@ hr.ngs.templater.example - csv-streaming-example + streaming-example ${project.version} diff --git a/src/test/java/DemoTests.java b/src/test/java/DemoTests.java index 57f64d84..3abfbea6 100644 --- a/src/test/java/DemoTests.java +++ b/src/test/java/DemoTests.java @@ -239,10 +239,9 @@ public void testSheets() throws Exception { SheetReportExample.main(null); } - @Test - public void testCsvStreaming() throws Exception { - CsvStreamingExample.main(null); + public void testStreaming() throws Exception { + StreamingExample.main(null); } @Test