fix native generation for large files

res/F.php

@@ -0,0 +1,129 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LeKoala;
+
+class F
+{
+    private const W = [
+        'cupiditate', 'praesentium', 'voluptas', 'pariatur',
+        'cum', 'lorem', 'ipsum', 'loquor', 'sic', 'amet'
+    ];
+    private const F = ['Julia', 'Lucius', 'Julius', 'Anna'];
+    private const S = ['Maximus', 'Corneli', 'Postumius', 'Servilius'];
+    private const C = ['US', 'NZ', 'FR', 'BE', 'NL', 'IT', 'UK'];
+    private const P = ['Roma', 'Caesera', 'Florentia', 'Lutetia'];
+    private const L = ['fr_FR', 'fr_BE', 'nl_BE', 'nl_NL', 'en_US', 'en_NZ', 'en_UK', 'it_IT'];
+
+    public static function pick(string $a, string $b): string
+    {
+        return random_int(0, 1) === 1 ? $a : $b;
+    }
+
+    public static function picka(array $arr, int $c = 1): array
+    {
+        $r = [];
+        while ($c > 0) {
+            $c--;
+            $r[] = $arr[array_rand($arr)];
+        }
+        return $r;
+    }
+
+    public static function d(): string
+    {
+        return date('Y-m-d', strtotime(self::pick('+', '-') . random_int(1, 365) . ' days'));
+    }
+
+    public static function t(): string
+    {
+        return sprintf('%02d:%02d:%02d', random_int(0, 23), random_int(0, 59), random_int(0, 59));
+    }
+
+    public static function dt(): string
+    {
+        return self::d() . ' ' . self::t();
+    }
+
+    public static function dtz(): string
+    {
+        return self::d() . 'T' . self::t() . 'Z';
+    }
+
+    public static function i(int $a = -100, int $b = 100): int
+    {
+        return random_int($a, $b);
+    }
+
+    public static function ctry(): string
+    {
+        return self::C[array_rand(self::C)];
+    }
+
+    public static function fn(): string
+    {
+        return self::F[array_rand(self::F)];
+    }
+
+    public static function sn(): string
+    {
+        return self::S[array_rand(self::S)];
+    }
+
+    public static function dom(): string
+    {
+        return self::W[array_rand(self::W)] . '.dev';
+    }
+
+    public static function w($a = 5, $b = 10): string
+    {
+        return implode(' ', self::picka(self::W, random_int($a, $b)));
+    }
+
+    public static function uw($a = 5, $b = 10): string
+    {
+        return ucfirst(self::w($a, $b));
+    }
+
+    public static function b(): bool
+    {
+        return (bool)random_int(0, 1);
+    }
+
+    public static function p(): string
+    {
+        return self::P[array_rand(self::P)];
+    }
+
+    public static function addr(): string
+    {
+        return 'via ' . self::w(1, 1) . ', ' . self::i(1, 20) . ' - ' . self::i(1000, 9999) . ' ' . self::p();
+    }
+
+    public static function l(string $ctry = null): string
+    {
+        do {
+            $l = self::L[array_rand(self::L)];
+        } while ($ctry !== null && !str_contains($l, $ctry));
+        return $l;
+    }
+
+    public static function lg(): string
+    {
+        return explode('_', self::l())[0];
+    }
+
+    public static function m(int $a = 10_000, int $b = 100_000): string
+    {
+        return number_format(self::i($a, $b)) . ' ' . self::pick('€', '$');
+    }
+
+    public static function em(string $p = null): string
+    {
+        if ($p === null) {
+            $p = self::fn();
+        }
+        return strtolower($p) . '@' . self::dom();
+    }
+}

src/Csv/Native.php

@@ -116,11 +116,7 @@ public function writeString(
 
         $stream = SpreadCompat::getMaxMemTempStream();
         $this->write($stream, $data);
-        rewind($stream);
-        $contents = stream_get_contents($stream);
-        if (!$contents) {
-            $contents = "";
-        }
+        $contents = SpreadCompat::getStreamContents($stream);
         fclose($stream);
         return $contents;
     }

src/SpreadCompat.php

@@ -115,6 +115,22 @@ public static function getExtensionForContent(string $contents): string
         return $ext;
     }
 
+    /**
+     * Don't forget fclose afterwards if you don't need the stream anymore
+     *
+     * @param resource $stream
+     */
+    public static function getStreamContents($stream): string
+    {
+        // Rewind to 0 before getting content from the start
+        rewind($stream);
+        $contents = stream_get_contents($stream);
+        if ($contents === false) {
+            $contents = "";
+        }
+        return $contents;
+    }
+
     /**
      * The memory limit of php://temp can be controlled by appending /maxmemory:NN,
      * where NN is the maximum amount of data to keep in memory before using a temporary file, in bytes.
@@ -124,6 +140,7 @@ public static function getExtensionForContent(string $contents): string
     public static function getMaxMemTempStream()
     {
         $mb = 4;
+        // Open for reading and writing; place the file pointer at the beginning of the file.
         $stream = fopen('php://temp/maxmemory:' . ($mb * 1024 * 1024), 'r+');
         if (!$stream) {
             throw new RuntimeException("Failed to open stream");
@@ -136,6 +153,8 @@ public static function getMaxMemTempStream()
      */
     public static function getOutputStream(string $filename = 'php://output')
     {
+        // Open for writing only; place the file pointer at the beginning of the file
+        // and truncate the file to zero length. If the file does not exist, attempt to create it.
         $stream = fopen($filename, 'w');
         if (!$stream) {
             throw new RuntimeException("Failed to open stream");
@@ -148,6 +167,7 @@ public static function getOutputStream(string $filename = 'php://output')
      */
     public static function getInputStream(string $filename)
     {
+        // Open for reading only; place the file pointer at the beginning of the file.
         $stream = fopen($filename, 'r');
         if (!$stream) {
             throw new RuntimeException("Failed to open stream");

src/Xlsx/Native.php

@@ -98,14 +98,20 @@ protected function write($zip, iterable $data): void
             'docProps/core.xml' => $this->genCoreXml(),
             'xl/styles.xml' => $this->genStyles(),
             'xl/workbook.xml' => $this->genWorkbook(),
-            'xl/worksheets/sheet1.xml' => $this->genWorksheet($data),
+            // 'xl/worksheets/sheet1.xml' => $this->genWorksheet($data),
             'xl/_rels/workbook.xml.rels' => $this->genWorkbookRels(),
             '[Content_Types].xml' => $this->genContentTypes(),
         ];
 
-        foreach ($allFiles as $path => $data) {
-            $zip->addFile($path, $data);
+        foreach ($allFiles as $path => $xml) {
+            $zip->addFile($path, $xml);
         }
+
+        // End up with worksheet
+        $stream =  $this->genWorksheet($data);
+        rewind($stream);
+        $zip->addFileFromStream('xl/worksheets/sheet1.xml', $stream);
+        fclose($stream);
     }
 
     protected function genRels(): string
@@ -221,57 +227,72 @@ protected function genWorkbook(): string
         // phpcs:enable
     }
 
-    protected function genWorksheet(iterable $data): string
+    /**
+     * @return resource
+     */
+    protected function genWorksheet(iterable $data)
     {
-        $rows = '';
+        $tempStream = SpreadCompat::getMaxMemTempStream();
         $r = 0;
 
+        // Since we don't know in advance, let's have the max
+        $MAX_ROW = 1048576;
+        $MAX_COL = 16384;
+
+        $maxCell = SpreadCompat::excelCell($MAX_ROW, $MAX_COL);
+
+        $header = <<<XML
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
+    xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
+    <dimension ref="A1:{$maxCell}"/>
+    <cols>
+        <col collapsed="false" hidden="false" max="1024" min="1" style="0" customWidth="false" width="11.5"/>
+    </cols>
+    <sheetData>
+XML;
+        fwrite($tempStream, $header);
+
         $dataRow = [""];
         foreach ($data as $dataRow) {
             $c = "";
             $i = 0;
             foreach ($dataRow as $k => $value) {
                 $cn = SpreadCompat::excelCell($r, $i);
 
-                // Auto detect numerical formats
-                if (
-                    !is_string($value)
-                    || $value == '0'
-                    || ($value[0] != '0' && ctype_digit($value))
-                    || preg_match("/^\-?(0|[1-9][0-9]*)(\.[0-9]+)?$/", $value)
-                ) {
-                    $c .= '<c r="' . $cn . '" t="n"><v>' . $value . '</v></c>'; //int,float,currency
+                if (!is_scalar($value) || $value === '') {
+                    $c .= '<c r="' . $cn . '"/>';
                 } else {
-                    $c .= '<c r="' . $cn . '" t="inlineStr"><is><t>' . self::esc($value) . '</t></is></c>';
+                    if (
+                        !is_string($value)
+                        || $value == '0'
+                        || ($value[0] != '0' && ctype_digit($value))
+                        || preg_match("/^\-?(0|[1-9][0-9]*)(\.[0-9]+)?$/", $value)
+                    ) {
+                        $c .= '<c r="' . $cn . '" t="n"><v>' . $value . '</v></c>'; //int,float,currency
+                    } else {
+                        $c .= '<c r="' . $cn . '" t="inlineStr"><is><t>' . self::esc($value) . '</t></is></c>';
+                    }
                 }
                 $c .= "\r\n";
-
                 $i++;
             }
 
             $r++;
-            $rows .= "<row r=\"$r\">$c</row>\r\n";
+            fwrite($tempStream, "<row r=\"$r\">$c</row>\r\n");
         }
 
-        $totalCols = count($dataRow);
-        $maxLetter = SpreadCompat::getLetter($totalCols);
-        $maxRow = $r;
 
-        // phpcs:disable
-        return <<<XML
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
-    xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
-    <dimension ref="A1:{$maxLetter}{$maxRow}"/>
-    <cols>
-        <col collapsed="false" hidden="false" max="1024" min="1" style="0" customWidth="false" width="11.5"/>
-    </cols>
-    <sheetData>
-        $rows
+        // $totalCols = count($dataRow);
+        // $maxLetter = SpreadCompat::getLetter($totalCols);
+        // $maxRow = $r;
+
+        $footer = <<<XML
     </sheetData>
 </worksheet>
 XML;
-        // phpcs:enable
+        fwrite($tempStream, $footer);
+        return $tempStream;
     }
 
     protected function genWorkbookRels(): string

test.php

@@ -4,6 +4,9 @@
 use Shuchkin\SimpleXLSXGen;
 
 require './vendor/autoload.php';
+require './res/F.php';
+
+use LeKoala\F;
 
 error_log(-1);
 
@@ -24,6 +27,34 @@
 $xlsx = SimpleXLSXGen::fromArray($books);
 // $xlsx->saveAs(__DIR__ . '/.dev/books.xlsx');
 
+// Short style faker data
+
+
+function gen($max = 1_000_000)
+{
+    $i = 0;
+    while ($i < $max) {
+        $i++;
+        yield [
+            $i,
+            F::d(),
+            F::dt(),
+            F::i(10_000, 30_000),
+            $fn = F::fn(),
+            $sn = F::sn(),
+            F::em($fn . $sn),
+            F::uw(5, 10),
+            F::addr(),
+            $ctry = F::ctry(),
+            F::l($ctry),
+            F::b(),
+            F::pick('1', ''),
+            F::m(),
+        ];
+    }
+}
+
 // Yes, you can stream the response directly
+// Even if it has 1 million rows and that it creates a file of 97 mb...
 $native = new Native();
-$native->output($books, 'books.xlsx');
+$native->output(gen(), 'books.xlsx');