From 1e38c837e4eee40c6d5c5b088e8ed1b512e02980 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 7 Sep 2023 13:28:39 +0200 Subject: [PATCH] Un-vendor tar (#4012) --- lib/src/io.dart | 4 +- lib/src/third_party/tar/CHANGELOG.md | 94 -- lib/src/third_party/tar/LICENSE | 21 - lib/src/third_party/tar/README.md | 214 ---- lib/src/third_party/tar/analysis_options.yaml | 18 - .../third_party/tar/lib/src/charcodes.dart | 76 -- .../third_party/tar/lib/src/constants.dart | 188 --- lib/src/third_party/tar/lib/src/entry.dart | 64 -- .../third_party/tar/lib/src/exception.dart | 19 - lib/src/third_party/tar/lib/src/format.dart | 322 ------ lib/src/third_party/tar/lib/src/header.dart | 409 ------- lib/src/third_party/tar/lib/src/reader.dart | 1012 ----------------- lib/src/third_party/tar/lib/src/sparse.dart | 150 --- lib/src/third_party/tar/lib/src/utils.dart | 588 ---------- lib/src/third_party/tar/lib/src/writer.dart | 499 -------- lib/src/third_party/tar/lib/tar.dart | 16 - lib/src/third_party/tar/vendored-pubspec.yaml | 20 - lib/src/third_party/vendor-state.yaml | 17 - pubspec.yaml | 2 +- test/io_test.dart | 2 +- test/test_pub.dart | 2 +- vendor.yaml | 6 - 22 files changed, 5 insertions(+), 3738 deletions(-) delete mode 100644 lib/src/third_party/tar/CHANGELOG.md delete mode 100644 lib/src/third_party/tar/LICENSE delete mode 100644 lib/src/third_party/tar/README.md delete mode 100644 lib/src/third_party/tar/analysis_options.yaml delete mode 100644 lib/src/third_party/tar/lib/src/charcodes.dart delete mode 100644 lib/src/third_party/tar/lib/src/constants.dart delete mode 100644 lib/src/third_party/tar/lib/src/entry.dart delete mode 100644 lib/src/third_party/tar/lib/src/exception.dart delete mode 100644 lib/src/third_party/tar/lib/src/format.dart delete mode 100644 lib/src/third_party/tar/lib/src/header.dart delete mode 100644 lib/src/third_party/tar/lib/src/reader.dart delete mode 100644 lib/src/third_party/tar/lib/src/sparse.dart delete mode 100644 lib/src/third_party/tar/lib/src/utils.dart delete mode 100644 lib/src/third_party/tar/lib/src/writer.dart delete mode 100644 lib/src/third_party/tar/lib/tar.dart delete mode 100644 lib/src/third_party/tar/vendored-pubspec.yaml delete mode 100644 lib/src/third_party/vendor-state.yaml delete mode 100644 vendor.yaml diff --git a/lib/src/io.dart b/lib/src/io.dart index 264db0c26..cc50e45a8 100644 --- a/lib/src/io.dart +++ b/lib/src/io.dart @@ -17,9 +17,9 @@ import 'package:http_multi_server/http_multi_server.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; import 'package:pool/pool.dart'; -// ignore: prefer_relative_imports -import 'package:pub/src/third_party/tar/lib/tar.dart'; import 'package:stack_trace/stack_trace.dart'; +// ignore: prefer_relative_imports +import 'package:tar/tar.dart'; import 'error_group.dart'; import 'exceptions.dart'; diff --git a/lib/src/third_party/tar/CHANGELOG.md b/lib/src/third_party/tar/CHANGELOG.md deleted file mode 100644 index 590a70db0..000000000 --- a/lib/src/third_party/tar/CHANGELOG.md +++ /dev/null @@ -1,94 +0,0 @@ -## 1.0.1 - -- Fix an incompatibility with Dart 3.1. - -## 1.0.0 - -- __Breaking__ Add class modifiers where applicable. - -## 0.5.6 - -- Allow cancelling a `TarEntry.contents` subscription before reading more files. - -## 0.5.5+1 - -- No user-visible changes. - -## 0.5.5 - -- Fix a crash when pausing a subscription to `TarEntry.contents` right before - it ends. - -## 0.5.4 - -- Fix generating corrupt tar files when adding lots of entries at very high - speeds [(#20)](https://github.com/simolus3/tar/issues/20). -- Allow tar files with invalid utf8 content in PAX header values if those - values aren't used for anything important. - -## 0.5.3 - -- Improve error messages when reading a tar entry after, or during, a call to - `moveNext()`. - -## 0.5.2 - -- This package now supports being compiled to JavaScript. - -## 0.5.1 - -- Improve performance when reading large archives - -## 0.5.0 - -- Support sync encoding with `tarConverter`. - -## 0.4.0 - -- Support generating tar files with GNU-style long link names - - Add `format` parameter to `tarWritingSink` and `tarWriterWith` - -## 0.3.3 - -- Drop `chunked_stream` dependency in favor of `package:async`. - -## 0.3.2 - -- Allow arbitrarily many zero bytes at the end of an archive when - `disallowTrailingData` is enabled. - -## 0.3.1 - -- Add `disallowTrailingData` parameter to `TarReader`. When the option is set, - `readNext` will ensure that the input stream does not emit further data after - the tar archive has been read fully. - -## 0.3.0 - -- Remove outdated references in the documentation - -## 0.3.0-nullsafety.0 - -- Remove `TarReader.contents` and `TarReader.header`. Use `current.contents` and `current.header`, respectively. -- Fix some minor implementation details - -## 0.2.0-nullsafety - -Most of the tar package has been rewritten, it's now based on the -implementation written by [Garett Tok Ern Liang](https://github.com/walnutdust) -in the GSoC 2020. - -- Added `tar` prefix to exported symbols. -- Remove `MemoryEntry`. Use `TarEntry.data` to create a tar entry from bytes. -- Make `WritingSink` private. Use `tarWritingSink` to create a general `StreamSink`. -- `TarReader` is now a [`StreamIterator`](https://api.dart.dev/stable/2.10.4/dart-async/StreamIterator-class.html), - the transformer had some design flaws. - -## 0.1.0-nullsafety.1 - -- Support writing user and group names -- Better support for PAX-headers and large files - -## 0.1.0-nullsafety.0 - -- Initial version diff --git a/lib/src/third_party/tar/LICENSE b/lib/src/third_party/tar/LICENSE deleted file mode 100644 index ed92ded99..000000000 --- a/lib/src/third_party/tar/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Simon Binder - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/lib/src/third_party/tar/README.md b/lib/src/third_party/tar/README.md deleted file mode 100644 index 8d5a33440..000000000 --- a/lib/src/third_party/tar/README.md +++ /dev/null @@ -1,214 +0,0 @@ -# tar - -![Build status](https://github.com/simolus3/tar/workflows/build/badge.svg) - -This package provides stream-based readers and writers for tar files. - -When working with large tar files, this library consumes considerably less memory -than [package:archive](https://pub.dev/packages/archive), although it is slightly slower due to the async overhead. - -## Reading - -To read entries from a tar file, use a `TarReader` with a `Stream` emitting bytes (as `List`): - -```dart -import 'dart:convert'; -import 'dart:io'; -import 'package:tar/tar.dart'; - -Future main() async { - final reader = TarReader(File('file.tar').openRead()); - - while (await reader.moveNext()) { - final entry = reader.current; - // Use reader.header to see the header of the current tar entry - print(entry.header.name); - // And reader.contents to read the content of the current entry as a stream - print(await entry.contents.transform(utf8.decoder).first); - } - // Note that the reader will automatically close if moveNext() returns false or - // throws. If you want to close a tar stream before that happens, use - // reader.cancel(); -} -``` - -To read `.tar.gz` files, transform the stream with `gzip.decoder` before -passing it to the `TarReader`. - -To easily go through all entries in a tar file, use `TarReader.forEach`: - -```dart -Future main() async { - final inputStream = File('file.tar').openRead(); - - await TarReader.forEach(inputStream, (entry) { - print(header.name); - print(await entry.contents.transform(utf8.decoder).first); - }); -} -``` - -__Warning__: Since the reader is backed by a single stream, concurrent calls to -`read` are not allowed! Similarly, if you're reading from an entry's `contents`, -make sure to fully drain the stream before calling `read()` again. -_Not_ subscribing to `contents` before calling `moveNext()` is acceptable too. -In this case, the reader will implicitly drain the stream. -The reader detects concurrency misuses and will throw an error when they occur, -there's no risk of reading faulty data. - -## Writing - -When writing archives, `package:tar` expects a `Stream` of tar entries to include in -the archive. -This stream can then be converted into a stream of byte-array chunks forming the -encoded tar archive. - -To write a tar stream into a `StreamSink>`, such as an `IOSink` returned by -`File.openWrite`, use `tarWritingSink`: - -```dart -import 'dart:convert'; -import 'dart:io'; -import 'package:tar/tar.dart'; - -Future main() async { - final output = File('test.tar').openWrite(); - final tarEntries = Stream.value( - TarEntry.data( - TarHeader( - name: 'hello.txt', - mode: int.parse('644', radix: 8), - ), - utf8.encode('Hello world'), - ), - ); - - await tarEntries.pipe(tarWritingSink(output)); -} -``` - -For more complex stream transformations, `tarWriter` can be used as a stream -transformer converting a stream of tar entries into archive bytes. - -Together with the `gzip.encoder` transformer from `dart:io`, this can be used -to write a `.tar.gz` file: - -```dart -import 'dart:io'; -import 'package:tar/tar.dart'; - -Future write(Stream entries) { - return entries - .transform(tarWriter) // convert entries into a .tar stream - .transform(gzip.encoder) // convert the .tar stream into a .tar.gz stream - .pipe(File('output.tar.gz').openWrite()); -} -``` - -A more complex example for writing files can be found in [`example/archive_self.dart`](example/archive_self.dart). - -### Encoding options - -By default, tar files are written in the pax format defined by the -POSIX.1-2001 specification (`--format=posix` in GNU tar). -When all entries have file names shorter than 100 chars and a size smaller -than 8 GB, this is equivalent to the `ustar` format. This library won't write -PAX headers when there is no reason to do so. -If you prefer writing GNU-style long filenames instead, you can use the -`format` option: - -```dart -Future write(Stream entries) { - return entries - .pipe( - tarWritingSink( - File('output.tar').openWrite(), - format: OutputFormat.gnuLongName, - )); -} -``` - -To change the output format on the `tarWriter` transformer, use -`tarWriterWith`. - -### Synchronous writing - -As the content of tar entries is defined as an asynchronous stream, the tar encoder is asynchronous too. -The more specific `SynchronousTarEntry` class stores tar content as a list of bytes, meaning that it can be -written synchronously. - -To synchronously write tar files, use `tarConverter` (or `tarConverterWith` for options): - -```dart -List createTarArchive(Iterable entries) { - late List result; - final sink = ByteConversionSink.withCallback((data) => result = data); - - final output = tarConverter.startChunkedConversion(sink); - entries.forEach(output.add); - output.close(); - - return result; -} -``` - -## Features - -- Supports v7, ustar, pax, gnu and star archives -- Supports extended pax headers for long file or link names -- Supports long file and link names generated by GNU-tar -- Hardened against denial-of-service attacks with invalid tar files -- Supports being compiled to JavaScript, tested on Node.js - -## Security considerations - -Internally, this package contains checks to guard against some invalid tar files. -In particular, - -- The reader doesn't allocate memory based on values in a tar file (so there's - a guard against DoS attacks with tar files containing huge headers). -- When encountering malformed tar files, the reader will throw a `TarException`. - Any other exception thrown indicates a bug in `package:tar` or how it's used. - The reader should never crash. -- Reading a tar file can be cancelled mid-stream without leaking resources. - -However, the tar reader __does not__ throw exceptions for wellformed archives -with suspicious contents, such as - -- File names beginning with `../`, `/` or names pointing out of the archive by - other means. -- Link references to files outside of the archive. -- Paths not using forward slashes. -- Gzip + tar bombs. -- Invalid permission bits in entries. -- ... - -When reading or extracting untrusted tar files, it is your responsibility to -detect and handle these cases. -For instance, this naive extraction function is susceptible to invalid tar -files containing paths outside of the target directory: - -```dart -Future extractTarGz(File tarGz, Directory target) async { - final input = tarGz.openRead().transform(gzip.decoder); - - await TarReader.forEach(input, (entry) async { - final destination = - // DON'T DO THIS! If `entry.name` contained `../`, this may escape the - // target directory. - path.joinAll([target.path, ...path.posix.split(entry.name)]); - - final f = File(destination); - await f.create(recursive: true); - await entry.contents.pipe(f.openWrite()); - }); -} -``` - -For an idea on how to guard against this, see the [extraction logic](https://github.com/dart-lang/pub/blob/3082796f8ba9b3f509265ac3a223312fb5033988/lib/src/io.dart#L904-L991) -used by the pub client. - ------ - -Big thanks to [Garett Tok Ern Liang](https://github.com/walnutdust) for writing the initial -Dart tar reader that this library is based on. diff --git a/lib/src/third_party/tar/analysis_options.yaml b/lib/src/third_party/tar/analysis_options.yaml deleted file mode 100644 index 1dd563f70..000000000 --- a/lib/src/third_party/tar/analysis_options.yaml +++ /dev/null @@ -1,18 +0,0 @@ -include: package:extra_pedantic/analysis_options.4.0.0.yaml - -analyzer: - language: - strict-inference: true - strict-raw-types: true - -linter: - rules: - close_sinks: false # This rule has just too many false-positives... - comment_references: true - package_api_docs: true - literal_only_boolean_expressions: false # Nothing wrong with a little while(true) - parameter_assignments: false - unnecessary_await_in_return: false - no_default_cases: false - prefer_asserts_with_message: false # We only use asserts for library-internal invariants - prefer_final_parameters: false # Too much noise diff --git a/lib/src/third_party/tar/lib/src/charcodes.dart b/lib/src/third_party/tar/lib/src/charcodes.dart deleted file mode 100644 index b25bb369a..000000000 --- a/lib/src/third_party/tar/lib/src/charcodes.dart +++ /dev/null @@ -1,76 +0,0 @@ -@internal -library; - -import 'package:meta/meta.dart'; - -/// "Line feed" control character. -const int $lf = 0x0a; - -/// Space character. -const int $space = 0x20; - -/// Character `0`. -const int $0 = 0x30; - -/// Character `1`. -const int $1 = 0x31; - -/// Character `2`. -const int $2 = 0x32; - -/// Character `3`. -const int $3 = 0x33; - -/// Character `4`. -const int $4 = 0x34; - -/// Character `5`. -const int $5 = 0x35; - -/// Character `6`. -const int $6 = 0x36; - -/// Character `7`. -const int $7 = 0x37; - -/// Character `8`. -const int $8 = 0x38; - -/// Character `9`. -const int $9 = 0x39; - -/// Character `<`. -const int $equal = 0x3d; - -/// Character `A`. -const int $A = 0x41; - -/// Character `K`. -const int $K = 0x4b; - -/// Character `L`. -const int $L = 0x4c; - -/// Character `S`. -const int $S = 0x53; - -/// Character `a`. -const int $a = 0x61; - -/// Character `g`. -const int $g = 0x67; - -/// Character `r`. -const int $r = 0x72; - -/// Character `s`. -const int $s = 0x73; - -/// Character `t`. -const int $t = 0x74; - -/// Character `u`. -const int $u = 0x75; - -/// Character `x`. -const int $x = 0x78; diff --git a/lib/src/third_party/tar/lib/src/constants.dart b/lib/src/third_party/tar/lib/src/constants.dart deleted file mode 100644 index d74a2b00f..000000000 --- a/lib/src/third_party/tar/lib/src/constants.dart +++ /dev/null @@ -1,188 +0,0 @@ -@internal -import 'dart:typed_data'; - -import 'package:meta/meta.dart'; - -import 'charcodes.dart'; -import 'exception.dart'; -import 'header.dart'; - -// Magic values to help us identify the TAR header type. -const magicGnu = [$u, $s, $t, $a, $r, $space]; // 'ustar ' -const versionGnu = [$space, 0]; // ' \x00' -const magicUstar = [$u, $s, $t, $a, $r, 0]; // 'ustar\x00' -const versionUstar = [$0, $0]; // '00' -const trailerStar = [$t, $a, $r, 0]; // 'tar\x00' - -/// Generates the corresponding [TypeFlag] associated with [byte]. -TypeFlag typeflagFromByte(int byte) { - switch (byte) { - case $0: - return TypeFlag.reg; - case 0: - return TypeFlag.regA; - case $1: - return TypeFlag.link; - case $2: - return TypeFlag.symlink; - case $3: - return TypeFlag.char; - case $4: - return TypeFlag.block; - case $5: - return TypeFlag.dir; - case $6: - return TypeFlag.fifo; - case $7: - return TypeFlag.reserved; - case $x: - return TypeFlag.xHeader; - case $g: - return TypeFlag.xGlobalHeader; - case $S: - return TypeFlag.gnuSparse; - case $L: - return TypeFlag.gnuLongName; - case $K: - return TypeFlag.gnuLongLink; - default: - if (64 < byte && byte < 91) { - return TypeFlag.vendor; - } - throw TarException.header('Invalid typeflag value $byte'); - } -} - -int typeflagToByte(TypeFlag flag) { - switch (flag) { - case TypeFlag.reg: - case TypeFlag.regA: - return $0; - case TypeFlag.link: - return $1; - case TypeFlag.symlink: - return $2; - case TypeFlag.char: - return $3; - case TypeFlag.block: - return $4; - case TypeFlag.dir: - return $5; - case TypeFlag.fifo: - return $6; - case TypeFlag.reserved: - return $7; - case TypeFlag.xHeader: - return $x; - case TypeFlag.xGlobalHeader: - return $g; - case TypeFlag.gnuSparse: - return $S; - case TypeFlag.gnuLongName: - return $L; - case TypeFlag.gnuLongLink: - return $K; - case TypeFlag.vendor: - throw ArgumentError("Can't write vendor-specific type-flags"); - } -} - -/// Keywords for PAX extended header records. -const paxPath = 'path'; -const paxLinkpath = 'linkpath'; -const paxSize = 'size'; -const paxUid = 'uid'; -const paxGid = 'gid'; -const paxUname = 'uname'; -const paxGname = 'gname'; -const paxMtime = 'mtime'; -const paxAtime = 'atime'; -const paxCtime = - 'ctime'; // Removed from later revision of PAX spec, but was valid -const paxComment = 'comment'; -const paxSchilyXattr = 'SCHILY.xattr.'; - -/// Keywords for GNU sparse files in a PAX extended header. -const paxGNUSparse = 'GNU.sparse.'; -const paxGNUSparseNumBlocks = 'GNU.sparse.numblocks'; -const paxGNUSparseOffset = 'GNU.sparse.offset'; -const paxGNUSparseNumBytes = 'GNU.sparse.numbytes'; -const paxGNUSparseMap = 'GNU.sparse.map'; -const paxGNUSparseName = 'GNU.sparse.name'; -const paxGNUSparseMajor = 'GNU.sparse.major'; -const paxGNUSparseMinor = 'GNU.sparse.minor'; -const paxGNUSparseSize = 'GNU.sparse.size'; -const paxGNUSparseRealSize = 'GNU.sparse.realsize'; - -/// A set of pax header keys supported by this library. -/// -/// The reader will ignore pax headers not listed in this map. -const supportedPaxHeaders = { - paxPath, - paxLinkpath, - paxSize, - paxUid, - paxGid, - paxUname, - paxGname, - paxMtime, - paxAtime, - paxCtime, - paxComment, - paxSchilyXattr, - paxGNUSparse, - paxGNUSparseNumBlocks, - paxGNUSparseOffset, - paxGNUSparseNumBytes, - paxGNUSparseMap, - paxGNUSparseName, - paxGNUSparseMajor, - paxGNUSparseMinor, - paxGNUSparseSize, - paxGNUSparseRealSize -}; - -/// User ID bit -const c_ISUID = 2048; - -/// Group ID bit -const c_ISGID = 1024; - -/// Sticky bit -const c_ISVTX = 512; - -/// Constants to determine file modes. -const modeType = 2401763328; -const modeSymLink = 134217728; -const modeDevice = 67108864; -const modeCharDevice = 2097152; -const modeNamedPipe = 33554432; -const modeSocket = 1677216; -const modeSetUid = 8388608; -const modeSetGid = 4194304; -const modeSticky = 1048576; -const modeDirectory = 2147483648; - -/// The offset of the checksum in the header -const checksumOffset = 148; -const checksumLength = 8; -const magicOffset = 257; -const versionOffset = 263; -const starTrailerOffset = 508; - -/// Size constants from various TAR specifications. -/// Size of each block in a TAR stream. -const blockSize = 512; -const blockSizeLog2 = 9; -const maxIntFor12CharOct = 0x1ffffffff; // 777 7777 7777 in oct - -const defaultSpecialLength = 4 * blockSize; - -/// Max length of the name field in USTAR format. -const nameSize = 100; - -/// Max length of the prefix field in USTAR format. -const prefixSize = 155; - -/// A full TAR block of zeros. -final zeroBlock = Uint8List(blockSize); diff --git a/lib/src/third_party/tar/lib/src/entry.dart b/lib/src/third_party/tar/lib/src/entry.dart deleted file mode 100644 index 7645bbb22..000000000 --- a/lib/src/third_party/tar/lib/src/entry.dart +++ /dev/null @@ -1,64 +0,0 @@ -import 'dart:async'; - -import 'header.dart'; - -/// An entry in a tar file. -/// -/// Usually, tar entries are read from a stream, and they're bound to the stream -/// from which they've been read. This means that they can only be read once, -/// and that only one [TarEntry] is active at a time. -final class TarEntry { - /// The parsed [TarHeader] of this tar entry. - final TarHeader header; - - /// The content stream of the active tar entry. - /// - /// For tar entries read through the reader provided by this library, - /// [contents] is a single-subscription streamed backed by the original stream - /// used to create the reader. - /// When listening on [contents], the stream needs to be fully drained before - /// the next call to [StreamIterator.moveNext]. It's acceptable to not listen - /// to [contents] at all before calling [StreamIterator.moveNext] again. - /// In that case, this library will take care of draining the stream to get to - /// the next entry. - final Stream> contents; - - /// The name of this entry, as indicated in the header or a previous pax - /// entry. - String get name => header.name; - - /// The type of tar entry (file, directory, etc.). - TypeFlag get type => header.typeFlag; - - /// The content size of this entry, in bytes. - int get size => header.size; - - /// Time of the last modification of this file, as indicated in the [header]. - DateTime get modified => header.modified; - - /// Creates a tar entry from a [header] and the [contents] stream. - /// - /// If the total length of [contents] is known, consider setting the - /// [header]'s [TarHeader.size] property to the appropriate value. - /// Otherwise, the tar writer needs to buffer contents to determine the right - /// size. - // factory so that this class can't be extended - factory TarEntry(TarHeader header, Stream> contents) = TarEntry._; - - TarEntry._(this.header, this.contents); - - /// Creates an in-memory tar entry from the [header] and the [data] to store. - static SynchronousTarEntry data(TarHeader header, List data) { - (header as HeaderImpl).size = data.length; - return SynchronousTarEntry._(header, data); - } -} - -/// A tar entry stored in memory. -final class SynchronousTarEntry extends TarEntry { - /// The contents of this tar entry as a byte array. - final List data; - - SynchronousTarEntry._(TarHeader header, this.data) - : super._(header, Stream.value(data)); -} diff --git a/lib/src/third_party/tar/lib/src/exception.dart b/lib/src/third_party/tar/lib/src/exception.dart deleted file mode 100644 index fa1fc924e..000000000 --- a/lib/src/third_party/tar/lib/src/exception.dart +++ /dev/null @@ -1,19 +0,0 @@ -import 'package:meta/meta.dart'; - -/// An exception indicating that there was an issue parsing a `.tar` file. -/// -/// The [message] contains reported from this exception contains details on the -/// location of the parsing error. -/// -/// This is the only exception that should be thrown by the `tar` package. Other -/// exceptions are either a bug in this package or errors thrown as a response -/// to API misuse. -final class TarException extends FormatException { - @internal - TarException(String message) : super(message); - - @internal - factory TarException.header(String message) { - return TarException('Invalid header: $message'); - } -} diff --git a/lib/src/third_party/tar/lib/src/format.dart b/lib/src/third_party/tar/lib/src/format.dart deleted file mode 100644 index b584f5c05..000000000 --- a/lib/src/third_party/tar/lib/src/format.dart +++ /dev/null @@ -1,322 +0,0 @@ -/// Holds the possible TAR formats that a file could take. -/// -/// This library supports the V7, USTAR, PAX, GNU, and STAR formats. The -/// [MaybeTarFormat] class generally describes any combination of those formats -/// and represents that we don't know the exact format yet. As soon as we do -/// know, the [TarFormat] enum represents the exact format of a header. -sealed class MaybeTarFormat { - /// The TAR formats are encoded in powers of two in [_value], such that we - /// can refine our guess via bit operations as we discover more information - /// about the TAR file. - /// A value of 0 means that the format is invalid. - int get _value; - - factory MaybeTarFormat._(int value) { - return switch (value) { - 1 => TarFormat.v7, - 2 => TarFormat.ustar, - 4 => TarFormat.pax, - 8 => TarFormat.gnu, - 16 => TarFormat.star, - final other => _MaybeTarFormat(other), - }; - } - - /// Returns a new [MaybeTarFormat] that signifies that it can be either - /// `this` or [other]'s format. - /// - /// **Example:** - /// ```dart - /// TarFormat format = TarFormat.ustar | TarFormat.pax; - /// ``` - /// - /// The above code would signify that we have limited `format` to either - /// the USTAR or PAX format, but need further information to refine the guess. - MaybeTarFormat operator |(TarFormat other); - - /// Returns if [other] is a possible resolution of `this`. - /// - /// For example, a [TarFormat] with a value of 6 means that we do not have - /// enough information to determine if it is [TarFormat.ustar] or - /// [TarFormat.pax], so either of them could be possible resolutions of - /// `this`. - bool has(MaybeTarFormat other); - - /// Returns a new [TarFormat] that signifies that it can only be [other]'s - /// format. - /// - /// **Example:** - /// ```dart - /// TarFormat format = TarFormat.PAX | TarFormat.USTAR; - /// ... - /// format = format.mayOnlyBe(TarFormat.USTAR); - /// ``` - /// - /// In the above example, we found that `format` could either be PAX or USTAR, - /// but later learnt that it can only be the USTAR format. - /// - /// If `has(other) == false`, [mayOnlyBe] will result in an unknown - /// [TarFormat]. - MaybeTarFormat mayOnlyBe(MaybeTarFormat other); - - /// Returns if this format might be valid. - /// - /// This returns true as well even if we have yet to fully determine what the - /// format is. - bool get valid; -} - -/// A fully resolved tar format. -/// -/// When we know that a tar entry must use a specific format, this is represented -/// with a value from this [TarFormat] enum. -enum TarFormat implements MaybeTarFormat { - /// Original Unix Version 7 (V7) AT&T tar tool prior to standardization. - /// - /// The structure of the V7 Header consists of the following: - /// - /// Start | End | Field - /// ========================================================================= - /// 0 | 100 | Path name, stored as null-terminated string. - /// 100 | 108 | File mode, stored as an octal number in ASCII. - /// 108 | 116 | User id of owner, as octal number in ASCII. - /// 116 | 124 | Group id of owner, as octal number in ASCII. - /// 124 | 136 | Size of file, as octal number in ASCII. - /// 136 | 148 | Modification time of file, number of seconds from epoch, - /// stored as an octal number in ASCII. - /// 148 | 156 | Header checksum, stored as an octal number in ASCII. - /// 156 | 157 | Link flag, determines the kind of header. - /// 157 | 257 | Link name, stored as a string. - /// 257 | 512 | NUL pad. - /// - /// Unused bytes are set to NUL ('\x00')s - /// - /// Reference: - /// https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&format=html - /// https://www.gnu.org/software/tar/manual/html_chapter/tar_15.html#SEC188 - /// http://cdrtools.sourceforge.net/private/man/star/star.4.html - v7(1, 'V7'), - - /// USTAR (Unix Standard TAR) header format defined in POSIX.1-1988. - /// - /// The structure of the USTAR Header consists of the following: - /// - /// Start | End | Field - /// ========================================================================= - /// 0 | 100 | Path name, stored as null-terminated string. - /// 100 | 108 | File mode, stored as an octal number in ASCII. - /// 108 | 116 | User id of owner, as octal number in ASCII. - /// 116 | 124 | Group id of owner, as octal number in ASCII. - /// 124 | 136 | Size of file, as octal number in ASCII. - /// 136 | 148 | Modification time of file, number of seconds from epoch, - /// stored as an octal number in ASCII. - /// 148 | 156 | Header checksum, stored as an octal number in ASCII. - /// 156 | 157 | Type flag, determines the kind of header. - /// Note that the meaning of the size field depends on the type. - /// 157 | 257 | Link name, stored as a string. - /// 257 | 263 | Contains the magic value "ustar\x00" to indicate that this is - /// the USTAR format. Full compliance requires user name and - /// group name fields to be set. - /// 263 | 265 | Version. "00" for POSIX standard archives. - /// 265 | 297 | User name, as null-terminated ASCII string. - /// 297 | 329 | Group name, as null-terminated ASCII string. - /// 329 | 337 | Major number for character or block device entry. - /// 337 | 345 | Minor number for character or block device entry. - /// 345 | 500 | Prefix. If the pathname is too long to fit in the 100 bytes - /// provided at the start, it can be split at any / character - /// with the first portion going here. - /// 500 | 512 | NUL pad. - /// - /// Unused bytes are set to NUL ('\x00')s - /// - /// User and group names should be used in preference to uid/gid values when - /// they are set and the corresponding names exist on the system. - /// - /// While this format is compatible with most tar readers, the format has - /// several limitations making it unsuitable for some usages. Most notably, it - /// cannot support sparse files, files larger than 8GiB, filenames larger than - /// 256 characters, and non-ASCII filenames. - /// - /// Reference: - /// https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&format=html - /// https://www.gnu.org/software/tar/manual/html_chapter/tar_15.html#SEC188 - /// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 - ustar(2, 'USTAR'), - - /// PAX header format defined in POSIX.1-2001. - /// - /// PAX extends USTAR by writing a special file with either the `x` or `g` - /// type flags to allow for attributes that are not conveniently stored in a - /// POSIX ustar archive to be held. - /// - /// Some newer formats add their own extensions to PAX by defining their - /// own keys and assigning certain semantic meaning to the associated values. - /// For example, sparse file support in PAX is implemented using keys - /// defined by the GNU manual (e.g., "GNU.sparse.map"). - /// - /// Reference: - /// https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&format=html - /// https://www.gnu.org/software/tar/manual/html_chapter/tar_15.html#SEC188 - /// http://cdrtools.sourceforge.net/private/man/star/star.4.html - /// http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html - pax(4, 'PAX'), - - /// GNU header format. - /// - /// The GNU header format is older than the USTAR and PAX standards and - /// is not compatible with them. The GNU format supports - /// arbitrary file sizes, filenames of arbitrary encoding and length, - /// sparse files, and other features. - /// - /// Start | End | Field - /// ========================================================================= - /// 0 | 100 | Path name, stored as null-terminated string. - /// 100 | 108 | File mode, stored as an octal number in ASCII. - /// 108 | 116 | User id of owner, as octal number in ASCII. - /// 116 | 124 | Group id of owner, as octal number in ASCII. - /// 124 | 136 | Size of file, as octal number in ASCII. - /// 136 | 148 | Modification time of file, number of seconds from epoch, - /// stored as an octal number in ASCII. - /// 148 | 156 | Header checksum, stored as an octal number in ASCII. - /// 156 | 157 | Type flag, determines the kind of header. - /// Note that the meaning of the size field depends on the type. - /// 157 | 257 | Link name, stored as a string. - /// 257 | 263 | Contains the magic value "ustar " to indicate that this is - /// the GNU format. - /// 263 | 265 | Version. " \x00" for POSIX standard archives. - /// 265 | 297 | User name, as null-terminated ASCII string. - /// 297 | 329 | Group name, as null-terminated ASCII string. - /// 329 | 337 | Major number for character or block device entry. - /// 337 | 345 | Minor number for character or block device entry. - /// 345 | 357 | Last Access time of file, number of seconds from epoch, - /// stored as an octal number in ASCII. - /// 357 | 369 | Last Changed time of file, number of seconds from epoch, - /// stored as an octal number in ASCII. - /// 369 | 381 | Offset - not used. - /// 381 | 385 | Longnames - deprecated - /// 385 | 386 | Unused. - /// 386 | 482 | Sparse data - 4 sets of (offset, numbytes) stored as - /// octal numbers in ASCII. - /// 482 | 483 | isExtended - if this field is non-zero, this header is - /// followed by additional sparse records, which are in the - /// same format as above. - /// 483 | 495 | Binary representation of the file's complete size, inclusive - /// of the sparse data. - /// 495 | 512 | NUL pad. - /// - /// It is recommended that PAX be chosen over GNU unless the target - /// application can only parse GNU formatted archives. - /// - /// Reference: - /// https://www.gnu.org/software/tar/manual/html_node/Standard.html - gnu(8, 'GNU'), - - /// Schily's TAR format, which is incompatible with USTAR. - /// This does not cover STAR extensions to the PAX format; these fall under - /// the PAX format. - /// - /// Start | End | Field - /// ========================================================================= - /// 0 | 100 | Path name, stored as null-terminated string. - /// 100 | 108 | File mode, stored as an octal number in ASCII. - /// 108 | 116 | User id of owner, as octal number in ASCII. - /// 116 | 124 | Group id of owner, as octal number in ASCII. - /// 124 | 136 | Size of file, as octal number in ASCII. - /// 136 | 148 | Modification time of file, number of seconds from epoch, - /// stored as an octal number in ASCII. - /// 148 | 156 | Header checksum, stored as an octal number in ASCII. - /// 156 | 157 | Type flag, determines the kind of header. - /// Note that the meaning of the size field depends on the type. - /// 157 | 257 | Link name, stored as a string. - /// 257 | 263 | Contains the magic value "ustar\x00" to indicate that this is - /// the GNU format. - /// 263 | 265 | Version. "00" for STAR archives. - /// 265 | 297 | User name, as null-terminated ASCII string. - /// 297 | 329 | Group name, as null-terminated ASCII string. - /// 329 | 337 | Major number for character or block device entry. - /// 337 | 345 | Minor number for character or block device entry. - /// 345 | 476 | Prefix. If the pathname is too long to fit in the 100 bytes - /// provided at the start, it can be split at any / character - /// with the first portion going here. - /// 476 | 488 | Last Access time of file, number of seconds from epoch, - /// stored as an octal number in ASCII. - /// 488 | 500 | Last Changed time of file, number of seconds from epoch, - /// stored as an octal number in ASCII. - /// 500 | 508 | NUL pad. - /// 508 | 512 | Trailer - "tar\x00". - /// - /// Reference: - /// http://cdrtools.sourceforge.net/private/man/star/star.4.html - star(16, 'STAR'), - ; - - @override - final int _value; - - final String _name; - - const TarFormat(this._value, this._name); - - @override - bool get valid => true; - - @override - MaybeTarFormat operator |(TarFormat other) { - return other == this ? this : _MaybeTarFormat(_value | other._value); - } - - @override - bool has(MaybeTarFormat other) { - return other == this; - } - - @override - MaybeTarFormat mayOnlyBe(MaybeTarFormat other) { - return MaybeTarFormat._(_value & other._value); - } - - @override - String toString() => _name; -} - -final class _MaybeTarFormat implements MaybeTarFormat { - // Note: We never represent a single tar format in a _MaybeTarFormat, these - // are represented in the TarFormat enum. - @override - final int _value; - - const _MaybeTarFormat(this._value); - - @override - int get hashCode => _value; - - @override - bool operator ==(Object other) { - if (other is! TarFormat) return false; - - return _value == other._value; - } - - @override - String toString() { - if (!valid) return 'Invalid'; - - return TarFormat.values.where(has).map((e) => e._name).join(' or '); - } - - @override - bool has(MaybeTarFormat other) => _value & other._value != 0; - - @override - bool get valid => _value != 0; - - @override - MaybeTarFormat mayOnlyBe(MaybeTarFormat other) { - return MaybeTarFormat._(_value & other._value); - } - - @override - MaybeTarFormat operator |(TarFormat other) { - return MaybeTarFormat._(_value | other._value); - } -} diff --git a/lib/src/third_party/tar/lib/src/header.dart b/lib/src/third_party/tar/lib/src/header.dart deleted file mode 100644 index e18cddf57..000000000 --- a/lib/src/third_party/tar/lib/src/header.dart +++ /dev/null @@ -1,409 +0,0 @@ -import 'dart:typed_data'; - -import 'package:meta/meta.dart'; - -import 'constants.dart'; -import 'exception.dart'; -import 'format.dart'; -import 'utils.dart'; - -/// Type flags for [TarHeader]. -/// -/// The type flag of a header indicates the kind of file associated with the -/// entry. This enum contains the various type flags over the different TAR -/// formats, and users should be careful that the type flag corresponds to the -/// TAR format they are working with. -enum TypeFlag { - /// [reg] indicates regular files. - /// - /// Old tar implementations have a seperate `TypeRegA` value. This library - /// will transparently read those as [regA]. - reg, - - /// Legacy-version of [reg] in old tar implementations. - /// - /// This is only used internally. - regA, - - /// Hard link - header-only, may not have a data body - link, - - /// Symbolic link - header-only, may not have a data body - symlink, - - /// Character device node - header-only, may not have a data body - char, - - /// Block device node - header-only, may not have a data body - block, - - /// Directory - header-only, may not have a data body - dir, - - /// FIFO node - header-only, may not have a data body - fifo, - - /// Currently does not have any meaning, but is reserved for the future. - reserved, - - /// Used by the PAX format to store key-value records that are only relevant - /// to the next file. - /// - /// This package transparently handles these types. - xHeader, - - /// Used by the PAX format to store key-value records that are relevant to all - /// subsequent files. - /// - /// This package only supports parsing and composing such headers, - /// but does not currently support persisting the global state across files. - xGlobalHeader, - - /// Indiates a sparse file in the GNU format - gnuSparse, - - /// Used by the GNU format for a meta file to store the path or link name for - /// the next file. - /// This package transparently handles these types. - gnuLongName, - gnuLongLink, - - /// Vendor specific typeflag, as defined in POSIX.1-1998. Seen as outdated but - /// may still exist on old files. - /// - /// This library uses a single enum to catch them all. - vendor -} - -/// Header of a tar entry -/// -/// A tar header stores meta-information about the matching tar entry, such as -/// its name. -sealed class TarHeader { - /// Type of header entry. In the V7 TAR format, this field was known as the - /// link flag. - TypeFlag get typeFlag; - - /// Name of file or directory entry. - String get name; - - /// Target name of link (valid for hard links or symbolic links). - String? get linkName; - - /// Permission and mode bits. - int get mode; - - /// User ID of owner. - int get userId; - - /// Group ID of owner. - int get groupId; - - /// User name of owner. - String? get userName; - - /// Group name of owner. - String? get groupName; - - /// Logical file size in bytes. - int get size; - - /// The time of the last change to the data of the TAR file. - DateTime get modified; - - /// The time of the last access to the data of the TAR file. - DateTime? get accessed; - - /// The time of the last change to the data or metadata of the TAR file. - DateTime? get changed; - - /// Major device number - int get devMajor; - - /// Minor device number - int get devMinor; - - /// The TAR format of the header. - /// - /// When this library is sure it knows the format of the tar entry, this will - /// be a [TarFormat] enum value. In other cases, a [MaybeTarFormat] could - /// represent multiple possible formats. - MaybeTarFormat get format; - - /// Checks if this header indicates that the file will have content. - bool get hasContent { - switch (typeFlag) { - case TypeFlag.link: - case TypeFlag.symlink: - case TypeFlag.block: - case TypeFlag.dir: - case TypeFlag.char: - case TypeFlag.fifo: - return false; - default: - return true; - } - } - - /// Creates a tar header from the individual field. - factory TarHeader({ - required String name, - TarFormat? format, - TypeFlag? typeFlag, - DateTime? modified, - String? linkName, - int mode = 0, - int size = -1, - String? userName, - int userId = 0, - int groupId = 0, - String? groupName, - DateTime? accessed, - DateTime? changed, - int devMajor = 0, - int devMinor = 0, - }) { - return HeaderImpl.internal( - name: name, - modified: modified ?? DateTime.fromMillisecondsSinceEpoch(0), - format: format ?? TarFormat.pax, - typeFlag: typeFlag ?? TypeFlag.reg, - linkName: linkName, - mode: mode, - size: size, - userName: userName, - userId: userId, - groupId: groupId, - groupName: groupName, - accessed: accessed, - changed: changed, - devMajor: devMajor, - devMinor: devMinor, - ); - } - - TarHeader._(); -} - -@internal -class HeaderImpl extends TarHeader { - TypeFlag internalTypeFlag; - - @override - String name; - - @override - String? linkName; - - @override - int mode; - - @override - int userId; - - @override - int groupId; - - @override - String? userName; - - @override - String? groupName; - - @override - int size; - - @override - DateTime modified; - - @override - DateTime? accessed; - - @override - DateTime? changed; - - @override - int devMajor; - - @override - int devMinor; - - @override - MaybeTarFormat format; - - @override - TypeFlag get typeFlag { - return switch (internalTypeFlag) { - TypeFlag.regA => TypeFlag.reg, // normalize - final other => other, - }; - } - - /// This constructor is meant to help us deal with header-only headers (i.e. - /// meta-headers that only describe the next file instead of being a header - /// to files themselves) - HeaderImpl.internal({ - required this.name, - required this.modified, - required this.format, - required TypeFlag typeFlag, - this.linkName, - this.mode = 0, - this.size = -1, - this.userName, - this.userId = 0, - this.groupId = 0, - this.groupName, - this.accessed, - this.changed, - this.devMajor = 0, - this.devMinor = 0, - }) : internalTypeFlag = typeFlag, - super._(); - - factory HeaderImpl.parseBlock(Uint8List headerBlock, - {Map paxHeaders = const {}}) { - assert(headerBlock.length == 512); - - final format = _getFormat(headerBlock); - final size = paxHeaders.size ?? headerBlock.readOctal(124, 12); - - // Start by reading data available in every format. - final header = HeaderImpl.internal( - format: format, - name: headerBlock.readString(0, 100), - mode: headerBlock.readOctal(100, 8), - // These should be octal, but some weird tar implementations ignore that?! - // Encountered with package:RAL, version 1.28.0 on pub - userId: headerBlock.readNumeric(108, 8), - groupId: headerBlock.readNumeric(116, 8), - size: size, - modified: secondsSinceEpoch(headerBlock.readOctal(136, 12)), - typeFlag: typeflagFromByte(headerBlock[156]), - linkName: headerBlock.readStringOrNullIfEmpty(157, 100), - ); - - if (header.hasContent && size < 0) { - throw TarException.header('Indicates an invalid size of $size'); - } - - if (format.valid && format != TarFormat.v7) { - // If it's a valid header that is not of the v7 format, it will have the - // USTAR fields - header - ..userName ??= headerBlock.readStringOrNullIfEmpty(265, 32) - ..groupName ??= headerBlock.readStringOrNullIfEmpty(297, 32) - ..devMajor = headerBlock.readNumeric(329, 8) - ..devMinor = headerBlock.readNumeric(337, 8); - - // Prefix to the file name - var prefix = ''; - if (format.has(TarFormat.ustar) || format.has(TarFormat.pax)) { - prefix = headerBlock.readString(345, 155); - - if (headerBlock.any(isNotAscii)) { - header.format = format.mayOnlyBe(TarFormat.pax); - } - } else if (format.has(TarFormat.star)) { - prefix = headerBlock.readString(345, 131); - header - ..accessed = secondsSinceEpoch(headerBlock.readNumeric(476, 12)) - ..changed = secondsSinceEpoch(headerBlock.readNumeric(488, 12)); - } else if (format.has(TarFormat.gnu)) { - header.format = TarFormat.gnu; - - if (headerBlock[345] != 0) { - header.accessed = secondsSinceEpoch(headerBlock.readNumeric(345, 12)); - } - - if (headerBlock[357] != 0) { - header.changed = secondsSinceEpoch(headerBlock.readNumeric(357, 12)); - } - } - - if (prefix.isNotEmpty) { - header.name = '$prefix/${header.name}'; - } - } - - return header.._applyPaxHeaders(paxHeaders); - } - - void _applyPaxHeaders(Map headers) { - for (final entry in headers.entries) { - if (entry.value == '') { - continue; // Keep the original USTAR value - } - - switch (entry.key) { - case paxPath: - name = entry.value; - break; - case paxLinkpath: - linkName = entry.value; - break; - case paxUname: - userName = entry.value; - break; - case paxGname: - groupName = entry.value; - break; - case paxUid: - userId = parseInt(entry.value); - break; - case paxGid: - groupId = parseInt(entry.value); - break; - case paxAtime: - accessed = parsePaxTime(entry.value); - break; - case paxMtime: - modified = parsePaxTime(entry.value); - break; - case paxCtime: - changed = parsePaxTime(entry.value); - break; - case paxSize: - size = parseInt(entry.value); - break; - default: - break; - } - } - } -} - -/// Checks that [rawHeader] represents a valid tar header based on the -/// checksum, and then attempts to guess the specific format based -/// on magic values. If the checksum fails, then an error is thrown. -MaybeTarFormat _getFormat(Uint8List rawHeader) { - final checksum = rawHeader.readOctal(checksumOffset, checksumLength); - - // Modern TAR archives use the unsigned checksum, but we check the signed - // checksum as well for compatibility. - if (checksum != rawHeader.computeUnsignedHeaderChecksum() && - checksum != rawHeader.computeSignedHeaderChecksum()) { - throw TarException.header('Checksum does not match'); - } - - final hasUstarMagic = rawHeader.matchesHeader(magicUstar); - if (hasUstarMagic) { - return rawHeader.matchesHeader(trailerStar, offset: starTrailerOffset) - ? TarFormat.star - : TarFormat.ustar | TarFormat.pax; - } - - if (rawHeader.matchesHeader(magicGnu) && - rawHeader.matchesHeader(versionGnu, offset: versionOffset)) { - return TarFormat.gnu; - } - - return TarFormat.v7; -} - -extension _ReadPaxHeaders on Map { - int? get size { - final sizeStr = this[paxSize]; - return sizeStr == null ? null : int.tryParse(sizeStr); - } -} diff --git a/lib/src/third_party/tar/lib/src/reader.dart b/lib/src/third_party/tar/lib/src/reader.dart deleted file mode 100644 index c16715159..000000000 --- a/lib/src/third_party/tar/lib/src/reader.dart +++ /dev/null @@ -1,1012 +0,0 @@ -import 'dart:async'; -import 'dart:collection'; -import 'dart:convert'; -import 'dart:typed_data'; - -import 'package:meta/meta.dart'; -import 'package:typed_data/typed_data.dart'; - -import 'charcodes.dart'; -import 'constants.dart'; -import 'entry.dart'; -import 'exception.dart'; -import 'format.dart'; -import 'header.dart'; -import 'sparse.dart'; -import 'utils.dart'; - -/// [TarReader] provides sequential access to the TAR files in a TAR archive. -/// It is designed to read from a stream and to spit out substreams for -/// individual file contents in order to minimize the amount of memory needed -/// to read each archive where possible. -final class TarReader implements StreamIterator { - final BlockReader _reader; - final PaxHeaders _paxHeaders = PaxHeaders(); - final int _maxSpecialFileSize; - - TarEntry? _current; - _CurrentEntryStream? _currentStream; - - /// Whether we're in the process of reading tar headers. - bool _isReadingHeaders = false; - - /// Whether this tar reader is terminally done. - /// - /// That is the case if: - /// - [cancel] was called - /// - [moveNext] completed to `false` once. - /// - [moveNext] completed to an error - /// - an error was emitted through a tar entry's content stream - bool _isDone = false; - - /// Whether we should ensure that the stream emits no further data after the - /// end of the tar file was reached. - final bool _checkNoTrailingData; - - /// Creates a tar reader reading from the raw [tarStream]. - /// - /// The [disallowTrailingData] parameter can be enabled to assert that the - /// [tarStream] contains exactly one tar archive before ending. - /// When [disallowTrailingData] is disabled (which is the default), the reader - /// will automatically cancel its stream subscription when [moveNext] returns - /// `false`. - /// When it is enabled and a marker indicating the end of an archive is - /// encountered, [moveNext] will wait for further events on the stream. If - /// further data is received, a [TarException] will be thrown and the - /// subscription will be cancelled. Otherwise, [moveNext] effectively waits - /// for a done event, making a cancellation unecessary. - /// Depending on the input stream, cancellations may cause unintended - /// side-effects. In that case, [disallowTrailingData] can be used to ensure - /// that the stream is only cancelled if it emits an invalid tar file. - /// - /// The [maxSpecialFileSize] parameter can be used to limit the maximum length - /// of hidden entries in the tar stream. These entries include extended PAX - /// headers or long names in GNU tar. The content of those entries has to be - /// buffered in the parser to properly read the following tar entries. To - /// avoid memory-based denial-of-service attacks, this library limits their - /// maximum length. Changing the default of 2 KiB is rarely necessary. - TarReader(Stream> tarStream, - {int maxSpecialFileSize = defaultSpecialLength, - bool disallowTrailingData = false}) - : _reader = BlockReader(tarStream), - _checkNoTrailingData = disallowTrailingData, - _maxSpecialFileSize = maxSpecialFileSize; - - @override - TarEntry get current { - final current = _current; - - if (current == null) { - throw StateError('Invalid call to TarReader.current. \n' - 'Did you call and await next() and checked that it returned true?'); - } - - return current; - } - - /// Reads the tar stream up until the beginning of the next logical file. - /// - /// If such file exists, the returned future will complete with `true`. After - /// the future completes, the next tar entry will be evailable in [current]. - /// - /// If no such file exists, the future will complete with `false`. - /// The future might complete with an [TarException] if the tar stream is - /// malformed or ends unexpectedly. - /// If the future completes with `false` or an exception, the reader will - /// [cancel] itself and release associated resources. Thus, it is invalid to - /// call [moveNext] again in that case. - @override - Future moveNext() async { - await _prepareToReadHeaders(); - try { - return await _moveNextInternal(); - } on Object { - await cancel(); - rethrow; - } - } - - /// Consumes the stream up to the contents of the next logical tar entry. - /// Will cancel the underlying subscription when returning false, but not when - /// it throws. - Future _moveNextInternal() async { - // We're reading a new logical file, so clear the local pax headers - _paxHeaders.clearLocals(); - - var gnuLongName = ''; - var gnuLongLink = ''; - var eofAcceptable = true; - - var format = TarFormat.ustar | - TarFormat.pax | - TarFormat.gnu | - TarFormat.v7 | - TarFormat.star; - - HeaderImpl? nextHeader; - - // Externally, [moveNext] iterates through the tar archive as if it is a - // series of files. Internally, the tar format often uses fake "files" to - // add meta data that describes the next file. These meta data "files" - // should not normally be visible to the outside. As such, this loop - // iterates through one or more "header files" until it finds a - // "normal file". - while (true) { - final rawHeader = await _readFullBlock(allowEmpty: eofAcceptable); - - nextHeader = await _readHeader(rawHeader); - if (nextHeader == null) { - if (eofAcceptable) { - await _handleExpectedEof(); - return false; - } else { - _unexpectedEof(); - } - } - - // We're beginning to read a file, if the tar file ends now something is - // wrong - eofAcceptable = false; - format = format.mayOnlyBe(nextHeader.format); - - // Check for PAX/GNU special headers and files. - if (nextHeader.typeFlag == TypeFlag.xHeader || - nextHeader.typeFlag == TypeFlag.xGlobalHeader) { - format = format.mayOnlyBe(TarFormat.pax); - final paxHeaderSize = _checkSpecialSize(nextHeader.size); - - final rawPaxHeaders = - (await _readFullBlock(amount: numBlocks(paxHeaderSize))) - .sublistView(0, paxHeaderSize); - - _paxHeaders.readPaxHeaders( - rawPaxHeaders, nextHeader.typeFlag == TypeFlag.xGlobalHeader); - - // This is a meta header affecting the next header. - continue; - } else if (nextHeader.typeFlag == TypeFlag.gnuLongLink || - nextHeader.typeFlag == TypeFlag.gnuLongName) { - format = format.mayOnlyBe(TarFormat.gnu); - final size = _checkSpecialSize(nextHeader.size); - final realName = await _readFullBlock(amount: numBlocks(size)); - - final readName = realName.readString(0, realName.length); - if (nextHeader.typeFlag == TypeFlag.gnuLongName) { - gnuLongName = readName; - } else { - gnuLongLink = readName; - } - - // This is a meta header affecting the next header. - continue; - } else { - // The old GNU sparse format is handled here since it is technically - // just a regular file with additional attributes. - - if (gnuLongName.isNotEmpty) nextHeader.name = gnuLongName; - if (gnuLongLink.isNotEmpty) nextHeader.linkName = gnuLongLink; - - if (nextHeader.internalTypeFlag == TypeFlag.regA) { - /// Legacy archives use trailing slash for directories - if (nextHeader.name.endsWith('/')) { - nextHeader.internalTypeFlag = TypeFlag.dir; - } else { - nextHeader.internalTypeFlag = TypeFlag.reg; - } - } - - final content = await _handleFile(nextHeader, rawHeader); - - // Set the final guess at the format - if (format.has(TarFormat.ustar) && format.has(TarFormat.pax)) { - format = format.mayOnlyBe(TarFormat.ustar); - } - nextHeader.format = format; - - _current = TarEntry(nextHeader, content); - final currentStreams = _currentStream; - assert(currentStreams == null || - currentStreams.state == _EntryStreamState.preListen); - _isReadingHeaders = false; - return true; - } - } - } - - @override - Future cancel() async { - if (_isDone) return; - - _isDone = true; - _current = null; - _currentStream = null; - _isReadingHeaders = false; - - // Note: Calling cancel is safe when the stream has already been completed. - // It's a noop in that case, which is what we want. - return _reader.close(); - } - - /// Utility function for quickly iterating through all entries in [tarStream]. - static Future forEach(Stream> tarStream, - FutureOr Function(TarEntry entry) action) async { - final reader = TarReader(tarStream); - try { - while (await reader.moveNext()) { - await action(reader.current); - } - } finally { - await reader.cancel(); - } - } - - /// Ensures that this reader can safely read headers now. - /// - /// This methods prevents: - /// * concurrent calls to [moveNext] - /// * a call to [moveNext] while a stream is active: - /// * if [TarEntry.contents] has never been listened to, or if it has a - /// cancelled subscription, we drain the stream. - /// * otherwise, throws a [StateError] - Future _prepareToReadHeaders() async { - if (_isDone) { - throw StateError('Tried to call TarReader.moveNext() on a canceled ' - 'reader. \n' - 'Note that a reader is canceled when moveNext() throws or returns ' - 'false.'); - } - - if (_isReadingHeaders) { - throw StateError('Concurrent call to TarReader.moveNext() detected. \n' - 'Please await all calls to Reader.moveNext().'); - } - _isReadingHeaders = true; - - final underlyingStream = _currentStream; - if (underlyingStream != null) { - switch (underlyingStream.state) { - case _EntryStreamState.preListen: - await underlyingStream.drain(); - // The stream should reset when drained (we do this in _publishStream) - assert(_currentStream == null); - - break; - case _EntryStreamState.subscriptionActive: - throw StateError( - 'Illegal call to TarReader.moveNext() while a previous stream was ' - 'active.\n' - 'When listening to tar contents, make sure the stream is ' - 'complete or cancelled before calling TarReader.moveNext() again.', - ); - case _EntryStreamState.cancelled: - // ignore: cancel_subscriptions - final subscription = underlyingStream._sourceSubscription!; - - // Re-purpose the existing subscription to drain the stream - assert(subscription.isPaused); - - subscription - ..onData(null) - ..resume(); - - try { - await subscription.asFuture(); - } on Object { - await cancel(); - rethrow; - } finally { - // This also draines the stream - _currentStream = null; - } - - break; - case _EntryStreamState.done: - assert( - false, - 'Unreachable: There should not be a currentStream in a done state, ' - 'as the stream is no longer current at that point.', - ); - break; - } - } - } - - int _checkSpecialSize(int size) { - if (size > _maxSpecialFileSize) { - throw TarException( - 'TAR file contains hidden entry with an invalid size of $size.'); - } - - return size; - } - - /// Ater we detected the end of a tar file, optionally check for trailing - /// data. - Future _handleExpectedEof() async { - if (_checkNoTrailingData) { - // Trailing zeroes are okay, but don't allow any more data here. - Uint8List block; - - do { - block = await _reader.nextBlock(); - if (!block.isAllZeroes) { - throw TarException( - 'Illegal content after the end of the tar archive.'); - } - } while (block.length == blockSize); - // The stream is done when we couldn't read the full block. - } - - await cancel(); - } - - Never _unexpectedEof() { - throw TarException.header('Unexpected end of file'); - } - - /// Reads [amount] blocks from the input stream, or throws an exception if - /// the stream ends prematurely. - Future _readFullBlock({bool allowEmpty = false, int amount = 1}) { - final blocks = Uint8List(amount * blockSize); - var offset = 0; - - return _reader.nextBlocks(amount).forEach((chunk) { - blocks.setAll(offset, chunk); - offset += chunk.length; - }).then((void _) { - if (allowEmpty && offset == 0) { - return Uint8List(0); - } else if (offset < blocks.length) { - _unexpectedEof(); - } else { - return blocks; - } - }); - } - - /// Reads the next block header and assumes that the underlying reader - /// is already aligned to a block boundary. It returns the raw block of the - /// header in case further processing is required. - /// - /// EOF is hit when one of the following occurs: - /// * Exactly 0 bytes are read and EOF is hit. - /// * Exactly 1 block of zeros is read and EOF is hit. - /// * At least 2 blocks of zeros are read. - Future _readHeader(Uint8List rawHeader) async { - // Exactly 0 bytes are read and EOF is hit. - if (rawHeader.isEmpty) return null; - - if (rawHeader.isAllZeroes) { - rawHeader = await _reader.nextBlock(); - - // Exactly 1 block of zeroes is read and EOF is hit. - if (rawHeader.isEmpty) return null; - - if (rawHeader.isAllZeroes) { - // Two blocks of zeros are read - Normal EOF. - return null; - } - - throw TarException('Encountered a non-zero block after a zero block'); - } - - return HeaderImpl.parseBlock(rawHeader, paxHeaders: _paxHeaders); - } - - /// Creates a stream of the next entry's content - Future>> _handleFile( - HeaderImpl header, Uint8List rawHeader) async { - List? sparseData; - if (header.typeFlag == TypeFlag.gnuSparse) { - sparseData = await _readOldGNUSparseMap(header, rawHeader); - } else { - sparseData = await _readGNUSparsePAXHeaders(header); - } - - if (sparseData != null) { - if (header.hasContent && - !validateSparseEntries(sparseData, header.size)) { - throw TarException.header('Invalid sparse file header.'); - } - - final sparseHoles = invertSparseEntries(sparseData, header.size); - final sparseDataLength = - sparseData.fold(0, (value, element) => value + element.length); - - final streamBlockCount = numBlocks(sparseDataLength); - final safeStream = _publishStream( - _reader.nextBlocks(streamBlockCount), streamBlockCount * blockSize); - return sparseStream(safeStream, sparseHoles, header.size); - } else { - var size = header.size; - if (!header.hasContent) size = 0; - - if (size < 0) { - throw TarException.header('Invalid size ($size) detected!'); - } - - if (size == 0) { - return _publishStream(const Stream.empty(), 0); - } else { - final blockCount = numBlocks(header.size); - return _publishStream(_reader.nextBlocks(blockCount), header.size); - } - } - } - - /// Publishes an library-internal stream for users. - /// - /// This adds a check to ensure that the stream we're exposing has the - /// expected length. It also sets the [_currentStream] field and resets it - /// when it's done. - Stream> _publishStream(Stream stream, int length) { - // There can only be one content stream at a time. This precondition is - // checked by _prepareToReadHeaders. - assert(_currentStream == null); - - return _currentStream = _CurrentEntryStream(this, stream, length); - } - - /// Checks the PAX headers for GNU sparse headers. - /// If they are found, then this function reads the sparse map and returns it. - /// This assumes that 0.0 headers have already been converted to 0.1 headers - /// by the PAX header parsing logic. - Future?> _readGNUSparsePAXHeaders(HeaderImpl header) async { - /// Identify the version of GNU headers. - var isVersion1 = false; - final major = _paxHeaders[paxGNUSparseMajor]; - final minor = _paxHeaders[paxGNUSparseMinor]; - - final sparseMapHeader = _paxHeaders[paxGNUSparseMap]; - if (major == '0' && (minor == '0' || minor == '1') || - // assume 0.0 or 0.1 if no version header is set - sparseMapHeader != null && sparseMapHeader.isNotEmpty) { - isVersion1 = false; - } else if (major == '1' && minor == '0') { - isVersion1 = true; - } else { - // Unknown version that we don't support - return null; - } - - header.format |= TarFormat.pax; - - /// Update [header] from GNU sparse PAX headers. - final possibleName = _paxHeaders[paxGNUSparseName] ?? ''; - if (possibleName.isNotEmpty) { - header.name = possibleName; - } - - final possibleSize = - _paxHeaders[paxGNUSparseSize] ?? _paxHeaders[paxGNUSparseRealSize]; - - if (possibleSize != null && possibleSize.isNotEmpty) { - final size = int.tryParse(possibleSize, radix: 10); - if (size == null) { - throw TarException.header('Invalid PAX size ($possibleSize) detected'); - } - - header.size = size; - } - - // Read the sparse map according to the appropriate format. - if (isVersion1) { - return await _readGNUSparseMap1x0(); - } - - return _readGNUSparseMap0x1(header); - } - - /// Reads the sparse map as stored in GNU's PAX sparse format version 1.0. - /// The format of the sparse map consists of a series of newline-terminated - /// numeric fields. The first field is the number of entries and is always - /// present. Following this are the entries, consisting of two fields - /// (offset, length). This function must stop reading at the end boundary of - /// the block containing the last newline. - /// - /// Note that the GNU manual says that numeric values should be encoded in - /// octal format. However, the GNU tar utility itself outputs these values in - /// decimal. As such, this library treats values as being encoded in decimal. - Future> _readGNUSparseMap1x0() async { - var newLineCount = 0; - final block = Uint8Queue(); - - /// Ensures that [block] h as at least [n] tokens. - Future feedTokens(int n) async { - while (newLineCount < n) { - final newBlock = await _readFullBlock(); - if (newBlock.length < blockSize) { - throw TarException.header( - 'GNU Sparse Map does not have enough lines!'); - } - - block.addAll(newBlock); - newLineCount += newBlock.where((byte) => byte == $lf).length; - } - } - - /// Get the next token delimited by a newline. This assumes that - /// at least one newline exists in the buffer. - String nextToken() { - newLineCount--; - final nextNewLineIndex = block.indexOf($lf); - final result = block.sublist(0, nextNewLineIndex); - block.removeRange(0, nextNewLineIndex + 1); - return result.readString(0, nextNewLineIndex); - } - - await feedTokens(1); - - // Parse for the number of entries. - // Use integer overflow resistant math to check this. - final numEntriesString = nextToken(); - final numEntries = int.tryParse(numEntriesString); - if (numEntries == null || numEntries < 0 || 2 * numEntries < numEntries) { - throw TarException.header( - 'Invalid sparse map number of entries: $numEntriesString!'); - } - - // Parse for all member entries. - // [numEntries] is trusted after this since a potential attacker must have - // committed resources proportional to what this library used. - await feedTokens(2 * numEntries); - - final sparseData = []; - - for (var i = 0; i < numEntries; i++) { - final offsetToken = nextToken(); - final lengthToken = nextToken(); - - final offset = int.tryParse(offsetToken); - final length = int.tryParse(lengthToken); - - if (offset == null || length == null) { - throw TarException.header( - 'Failed to read a GNU sparse map entry. Encountered ' - 'offset: $offsetToken, length: $lengthToken'); - } - - sparseData.add(SparseEntry(offset, length)); - } - return sparseData; - } - - /// Reads the sparse map as stored in GNU's PAX sparse format version 0.1. - /// The sparse map is stored in the PAX headers and is stored like this: - /// `offset₀,size₀,offset₁,size₁...` - List _readGNUSparseMap0x1(TarHeader header) { - // Get number of entries, check for integer overflows - final numEntriesString = _paxHeaders[paxGNUSparseNumBlocks]; - final numEntries = - numEntriesString != null ? int.tryParse(numEntriesString) : null; - - if (numEntries == null || numEntries < 0 || 2 * numEntries < numEntries) { - throw TarException.header('Invalid GNU version 0.1 map'); - } - - // There should be two numbers in [sparseMap] for each entry. - final sparseMap = _paxHeaders[paxGNUSparseMap]?.split(','); - if (sparseMap == null) { - throw TarException.header('Invalid GNU version 0.1 map'); - } - - if (sparseMap.length != 2 * numEntries) { - throw TarException.header( - 'Detected sparse map length ${sparseMap.length} ' - 'that is not twice the number of entries $numEntries'); - } - - /// Loop through sparse map entries. - /// [numEntries] is now trusted. - final sparseData = []; - for (var i = 0; i < sparseMap.length; i += 2) { - final offset = int.tryParse(sparseMap[i]); - final length = int.tryParse(sparseMap[i + 1]); - - if (offset == null || length == null) { - throw TarException.header( - 'Failed to read a GNU sparse map entry. Encountered ' - 'offset: $offset, length: $length'); - } - - sparseData.add(SparseEntry(offset, length)); - } - - return sparseData; - } - - /// Reads the sparse map from the old GNU sparse format. - /// The sparse map is stored in the tar header if it's small enough. - /// If it's larger than four entries, then one or more extension headers are - /// used to store the rest of the sparse map. - /// - /// [TarHeader.size] does not reflect the size of any extended headers used. - /// Thus, this function will read from the chunked stream iterator to fetch - /// extra headers. - /// - /// See also: https://www.gnu.org/software/tar/manual/html_section/tar_94.html#SEC191 - Future> _readOldGNUSparseMap( - HeaderImpl header, Uint8List rawHeader) async { - // Make sure that the input format is GNU. - // Unfortunately, the STAR format also has a sparse header format that uses - // the same type flag but has a completely different layout. - if (header.format != TarFormat.gnu) { - throw TarException.header('Tried to read sparse map of non-GNU header'); - } - - // Read the real size of the file when sparse holes are expanded. - header.size = rawHeader.readNumeric(483, 12); - final sparseEntries = []; - - bool readEntry(Uint8List source, int offset) { - // If a sparse header starts with a null byte, it marks the end of the - // sparse structures. - if (rawHeader[offset] == 0) return false; - - final fileOffset = source.readNumeric(offset, 12); - final length = source.readNumeric(offset + 12, 12); - - sparseEntries.add(SparseEntry(fileOffset, length)); - return true; - } - - // The first four sparse headers are stored in the tar header itself - for (var i = 0; i < 4; i++) { - final offset = 386 + 24 * i; - if (!readEntry(rawHeader, offset)) break; - } - - var isExtended = rawHeader[482] != 0; - - while (isExtended) { - // Ok, we have a new block of sparse headers to process - final block = await _readFullBlock(); - - // A full block of sparse data contains up to 21 entries - for (var i = 0; i < 21; i++) { - if (!readEntry(block, i * 24)) break; - } - - // The last bytes indicates whether another sparse header block follows. - isExtended = block[504] != 0; - } - - return sparseEntries; - } -} - -@internal -final class PaxHeaders extends UnmodifiableMapBase { - final Map _globalHeaders = {}; - Map _localHeaders = {}; - - /// Applies new global PAX-headers from the map. - /// - /// The [headers] will replace global headers with the same key, but leave - /// others intact. - void newGlobals(Map headers) { - _globalHeaders.addAll(headers); - } - - /// Applies new local PAX-headers from the map. - /// - /// This replaces all currently active local headers. - void newLocals(Map headers) { - _localHeaders = headers; - } - - /// Clears local headers. - /// - /// This is used by the reader after a file has ended, as local headers only - /// apply to the next entry. - void clearLocals() { - _localHeaders = {}; - } - - @override - String? operator [](Object? key) { - return _localHeaders[key] ?? _globalHeaders[key]; - } - - @override - Iterable get keys => {..._globalHeaders.keys, ..._localHeaders.keys}; - - /// Decodes the content of an extended pax header entry. - /// - /// Semantically, a [PAX Header][posix pax] is a map with string keys and - /// values, where both keys and values are encodes with utf8. - /// - /// However, [old GNU Versions][gnu sparse00] used to repeat keys to store - /// sparse file information in sparse headers. This method will transparently - /// rewrite the PAX format of version 0.0 to version 0.1. - /// - /// [posix pax]: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_03 - /// [gnu sparse00]: https://www.gnu.org/software/tar/manual/html_section/tar_94.html#SEC192 - void readPaxHeaders(List data, bool isGlobal, - {bool ignoreUnknown = true}) { - var offset = 0; - final map = {}; - final sparseMap = []; - - Never error() => throw TarException.header('Invalid PAX record'); - - while (offset < data.length) { - // At the start of an entry, expect its length which is terminated by a - // space char. - final space = data.indexOf($space, offset); - if (space == -1) break; - - var length = 0; - var currentChar = data[offset]; - var charsInLength = 0; - while (currentChar >= $0 && currentChar <= $9) { - length = length * 10 + currentChar - $0; - charsInLength++; - currentChar = data[++offset]; - } - - if (length == 0) { - error(); - } - - // Skip the whitespace - if (currentChar != $space) { - error(); - } - offset++; - - // Length also includes the length description and a space we just read - final endOfEntry = offset + length - 1 - charsInLength; - // checking against endOfEntry - 1 because the trailing whitespace is - // optional for the last entry - if (endOfEntry < offset || endOfEntry - 1 > data.length) { - error(); - } - - // Read the key - final nextEquals = data.indexOf($equal, offset); - if (nextEquals == -1 || nextEquals >= endOfEntry) { - error(); - } - - final key = utf8.decoder.convert(data, offset, nextEquals); - // Skip over the equals sign - offset = nextEquals + 1; - - // Subtract one for trailing newline for value - final endOfValue = endOfEntry - 1; - - if (!_isValidPaxKey(key)) { - error(); - } - - // If we're seeing weird PAX Version 0.0 sparse keys, expect alternating - // GNU.sparse.offset and GNU.sparse.numbytes headers. - if (key == paxGNUSparseNumBytes || key == paxGNUSparseOffset) { - final value = utf8.decoder.convert(data, offset, endOfValue); - - if (!_isValidPaxRecord(key, value) || - (sparseMap.length.isEven && key != paxGNUSparseOffset) || - (sparseMap.length.isOdd && key != paxGNUSparseNumBytes) || - value.contains(',')) { - error(); - } - - sparseMap.add(value); - } else if (!ignoreUnknown || supportedPaxHeaders.contains(key)) { - // Ignore unrecognized headers to avoid unbounded growth of the global - // header map. - final value = unsafeUtf8Decoder.convert(data, offset, endOfValue); - - if (!_isValidPaxRecord(key, value)) { - error(); - } - - map[key] = value; - } - - // Skip over value - offset = endOfValue; - // and the trailing newline - final hasNewline = offset < data.length; - if (hasNewline && data[offset] != $lf) { - throw TarException('Invalid PAX Record (missing trailing newline)'); - } - offset++; - } - - if (sparseMap.isNotEmpty) { - map[paxGNUSparseMap] = sparseMap.join(','); - } - - if (isGlobal) { - newGlobals(map); - } else { - newLocals(map); - } - } - - // NB: Some Tar files have malformed UTF-8 data in the headers, we should - // decode them anyways even if they're broken - static const unsafeUtf8Decoder = Utf8Decoder(allowMalformed: true); - - static bool _isValidPaxKey(String key) { - // These limitations are documented in the PAX standard. - return key.isNotEmpty && !key.contains('=') & !key.codeUnits.contains(0); - } - - /// Checks whether [key], [value] is a valid entry in a pax header. - /// - /// This is adopted from the Golang tar reader (`validPAXRecord`), which says - /// that "Keys and values should be UTF-8, but the number of bad writers out - /// there forces us to be a more liberal." - static bool _isValidPaxRecord(String key, String value) { - // These aren't documented in any standard, but Golangs's tar has them and - // got away with it. - switch (key) { - case paxPath: - case paxLinkpath: - case paxUname: - case paxGname: - return !value.codeUnits.contains(0); - default: - return true; - } - } -} - -enum _EntryStreamState { - preListen, - subscriptionActive, - cancelled, - done, -} - -/// The underlying content stream for the [TarReader._current] entry. Draining -/// this stream will move the tar reader to the beginning of the next file. -/// -/// This is not the same as `_current.stream` for sparse files, which are -/// reported as expanded through [TarEntry.contents]. -/// For that reason, we prefer to drain this stream when skipping a tar entry. -/// When we know we're skipping data, there's no point expanding sparse holes. -/// -/// Draining this stream will set the [TarReader._currentStream] field back to -/// null. There can only be one content stream at the time. -final class _CurrentEntryStream extends Stream> { - _EntryStreamState state = _EntryStreamState.preListen; - - final TarReader _reader; - final Stream _source; - - final StreamController> _listener = StreamController(sync: true); - // ignore: cancel_subscriptions - StreamSubscription>? _sourceSubscription; - - int _remainingContentSize; - int _remainingPaddingSize; - bool _hadError = false; - bool _isInContent = true; - - _CurrentEntryStream(this._reader, this._source, this._remainingContentSize) - : _remainingPaddingSize = _paddingFor(_remainingContentSize); - - @override - StreamSubscription> listen(void Function(List event)? onData, - {Function? onError, void Function()? onDone, bool? cancelOnError}) { - // Make sure that this entry is still the current one: If users store the - // contents of a tar entry, then read more tar entries, and finally try to - // read the stream of the old contents, they'd get an exception about the - // stream already being listened to. - // This can be a bit confusing, so this check enables a better error UX. - if (_reader._currentStream != this) { - throw StateError( - 'Tried listening to an outdated tar entry. \n' - 'As all tar entries found by a reader are backed by a single source ' - 'stream, only the latest tar entry can be read. It looks like you ' - 'stored the results of `tarEntry.contents` somewhere, called ' - '`reader.moveNext()` and then read the contents of the previous ' - 'entry.\n' - 'For more details, including a discussion of workarounds, see ' - 'https://github.com/simolus3/tar/issues/18', - ); - } else if (state != _EntryStreamState.preListen) { - throw StateError( - 'A tar entry has been listened to multiple times. \n' - 'As all tar entries are read from what\'s likely a single-' - 'subscription stream, this is unsupported. If you didn\'t read a tar ' - 'entry multiple times yourself, perhaps you\'ve called `moveNext()` ' - 'before reading contents?', - ); - } - - // Now we have a listener, so - state = _EntryStreamState.subscriptionActive; - // ignore: cancel_subscriptions - final sub = _sourceSubscription = _source.listen( - _forwardData, - onError: _forwardError, - onDone: _forwardDone, - ); - - _listener - ..onPause = sub.pause - ..onResume = sub.resume - ..onCancel = () { - // Pause the source subscription. When reading the next entry, the tar - // reader will drain the remaining source stream. - sub.pause(); - state = _EntryStreamState.cancelled; - }; - - return _listener.stream.listen( - onData, - onError: onError, - onDone: onDone, - cancelOnError: cancelOnError, - ); - } - - static int _paddingFor(int contentSize) { - final offsetInLastBlock = contentSize.toUnsigned(blockSizeLog2); - if (offsetInLastBlock != 0) { - return blockSize - offsetInLastBlock; - } - return 0; - } - - void _assertInStateForForwarding() { - assert(state == _EntryStreamState.subscriptionActive && - _listener.hasListener && - !_listener.isPaused); - } - - void _forwardData(Uint8List event) { - _assertInStateForForwarding(); - - if (_isInContent) { - if (event.length <= _remainingContentSize) { - // We can fully add this chunk as it consists entirely of data - _listener.add(event); - _remainingContentSize -= event.length; - } else { - // We can add the first bytes as content, the others are padding that we - // shouldn't emit - _listener.add(event.sublistView(0, _remainingContentSize)); - _isInContent = false; - _remainingPaddingSize -= event.length - _remainingContentSize; - _remainingContentSize = 0; - } - } else { - // Ok, the entire event is padding - _remainingPaddingSize -= event.length; - } - - // The underlying stream comes from pkg:tar, so if we get too many bytes - // that's a bug in this package. - assert(_remainingPaddingSize >= 0, 'Stream emitted to many bytes'); - } - - void _forwardError(Object error, StackTrace trace) { - _assertInStateForForwarding(); - - _hadError = true; - _listener.addError(error, trace); - } - - void _forwardDone() { - _assertInStateForForwarding(); - - // Now that the source stream is done, reset the stream state on the reader. - state = _EntryStreamState.done; - _sourceSubscription = null; - _reader._currentStream = null; - - // If the stream stopped after an error, the user is already aware that - // something is wrong. - if (_remainingContentSize > 0 && !_hadError) { - _listener.addError( - TarException('Unexpected end of tar file'), StackTrace.current); - } - unawaited(_listener.close()); - } -} diff --git a/lib/src/third_party/tar/lib/src/sparse.dart b/lib/src/third_party/tar/lib/src/sparse.dart deleted file mode 100644 index bb938d0ac..000000000 --- a/lib/src/third_party/tar/lib/src/sparse.dart +++ /dev/null @@ -1,150 +0,0 @@ -@internal -import 'package:async/async.dart'; -import 'package:meta/meta.dart'; - -import 'exception.dart'; -import 'utils.dart'; - -/// Represents a [length]-sized fragment at [offset] in a file. -/// -/// [SparseEntry]s can represent either data or holes, and we can easily -/// convert between the two if we know the size of the file, all the sparse -/// data and all the sparse entries combined must give the full size. -final class SparseEntry { - final int offset; - final int length; - - SparseEntry(this.offset, this.length); - - int get end => offset + length; - - @override - String toString() => 'offset: $offset, length $length'; - - @override - bool operator ==(Object other) { - if (other is! SparseEntry) return false; - - return offset == other.offset && length == other.length; - } - - @override - int get hashCode => offset ^ length; -} - -/// Generates a stream of the sparse file contents of size [size], given -/// [sparseHoles] and the raw content in [source]. -@internal -Stream> sparseStream( - Stream> source, List sparseHoles, int size) { - if (sparseHoles.isEmpty) { - return ChunkedStreamReader(source).readStream(size); - } - - return _sparseStream(source, sparseHoles, size); -} - -/// Generates a stream of the sparse file contents of size [size], given -/// [sparseHoles] and the raw content in [source]. -/// -/// [sparseHoles] has to be non-empty. -Stream> _sparseStream( - Stream> source, List sparseHoles, int size) async* { - // Current logical position in sparse file. - var position = 0; - - // Index of the next sparse hole in [sparseHoles] to be processed. - var sparseHoleIndex = 0; - - // Iterator through [source] to obtain the data bytes. - final iterator = ChunkedStreamReader(source); - - while (position < size) { - // Yield all the necessary sparse holes. - while (sparseHoleIndex < sparseHoles.length && - sparseHoles[sparseHoleIndex].offset == position) { - final sparseHole = sparseHoles[sparseHoleIndex]; - yield* zeroes(sparseHole.length); - position += sparseHole.length; - sparseHoleIndex++; - } - - if (position == size) break; - - /// Yield up to the next sparse hole's offset, or all the way to the end - /// if there are no sparse holes left. - var yieldTo = size; - if (sparseHoleIndex < sparseHoles.length) { - yieldTo = sparseHoles[sparseHoleIndex].offset; - } - - // Yield data as substream, but make sure that we have enough data. - var checkedPosition = position; - await for (final chunk in iterator.readStream(yieldTo - position)) { - yield chunk; - checkedPosition += chunk.length; - } - - if (checkedPosition != yieldTo) { - throw TarException('Invalid sparse data: Unexpected end of input stream'); - } - - position = yieldTo; - } -} - -/// Reports whether [sparseEntries] is a valid sparse map. -/// It does not matter whether [sparseEntries] represents data fragments or -/// hole fragments. -bool validateSparseEntries(List sparseEntries, int size) { - // Validate all sparse entries. These are the same checks as performed by - // the BSD tar utility. - if (size < 0) return false; - - SparseEntry? previous; - - for (final current in sparseEntries) { - // Negative values are never okay. - if (current.offset < 0 || current.length < 0) return false; - - // Integer overflow with large length. - if (current.offset + current.length < current.offset) return false; - - // Region extends beyond the actual size. - if (current.end > size) return false; - - // Regions cannot overlap and must be in order. - if (previous != null && previous.end > current.offset) return false; - - previous = current; - } - - return true; -} - -/// Converts a sparse map ([source]) from one form to the other. -/// If the input is sparse holes, then it will output sparse datas and -/// vice-versa. The input must have been already validated. -/// -/// This function mutates [source] and returns a normalized map where: -/// * adjacent fragments are coalesced together -/// * only the last fragment may be empty -/// * the endOffset of the last fragment is the total size -List invertSparseEntries(List source, int size) { - final result = []; - var previous = SparseEntry(0, 0); - for (final current in source) { - /// Skip empty fragments - if (current.length == 0) continue; - - final newLength = current.offset - previous.offset; - if (newLength > 0) { - result.add(SparseEntry(previous.offset, newLength)); - } - - previous = SparseEntry(current.end, 0); - } - final lastLength = size - previous.offset; - result.add(SparseEntry(previous.offset, lastLength)); - return result; -} diff --git a/lib/src/third_party/tar/lib/src/utils.dart b/lib/src/third_party/tar/lib/src/utils.dart deleted file mode 100644 index 1c42cc6e2..000000000 --- a/lib/src/third_party/tar/lib/src/utils.dart +++ /dev/null @@ -1,588 +0,0 @@ -@internal -library; - -import 'dart:async'; -import 'dart:convert'; -import 'dart:math'; -import 'dart:typed_data'; - -import 'package:meta/meta.dart'; - -import 'charcodes.dart'; -import 'constants.dart'; -import 'exception.dart'; - -const _checksumEnd = checksumOffset + checksumLength; -const _checksumPlaceholder = $space; - -extension ByteBufferUtils on Uint8List { - String readString(int offset, int maxLength) { - return readStringOrNullIfEmpty(offset, maxLength) ?? ''; - } - - Uint8List sublistView(int start, [int? end]) { - return Uint8List.sublistView(this, start, end); - } - - String? readStringOrNullIfEmpty(int offset, int maxLength) { - var data = sublistView(offset, offset + maxLength); - var contentLength = data.indexOf(0); - // If there's no \0, assume that the string fills the whole segment - if (contentLength.isNegative) contentLength = maxLength; - - if (contentLength == 0) return null; - - data = data.sublistView(0, contentLength); - try { - return utf8.decode(data); - } on FormatException { - return String.fromCharCodes(data).trim(); - } - } - - /// Parse an octal string encoded from index [offset] with the maximum length - /// [length]. - int readOctal(int offset, int length) { - var result = 0; - var multiplier = 1; - - for (var i = length - 1; i >= 0; i--) { - final charCode = this[offset + i]; - // Some tar implementations add a \0 or space at the end, ignore that - if (charCode == 0 || charCode == $space) continue; - if (charCode < $0 || charCode > $9) { - throw TarException('Invalid octal value'); - } - - // Obtain the numerical value of this digit - final digit = charCode - $0; - result += digit * multiplier; - multiplier <<= 3; // Multiply by the base, 8 - } - - return result; - } - - /// Parses an encoded int, either as base-256 or octal. - /// - /// This function may return negative numbers. - int readNumeric(int offset, int length) { - if (length == 0) return 0; - - // Check for base-256 (binary) format first. If the first bit is set, then - // all following bits constitute a two's complement encoded number in big- - // endian byte order. - final firstByte = this[offset]; - if (firstByte & 0x80 != 0) { - // Handling negative numbers relies on the following identity: - // -a-1 == ~a - // - // If the number is negative, we use an inversion mask to invert the - // date bytes and treat the value as an unsigned number. - final inverseMask = firstByte & 0x40 != 0 ? 0xff : 0x00; - - // Ignore signal bit in the first byte - var x = (firstByte ^ inverseMask) & 0x7f; - - for (var i = 1; i < length; i++) { - var byte = this[offset + i]; - byte ^= inverseMask; - - x = x << 8 | byte; - } - - return inverseMask == 0xff ? ~x : x; - } - - return readOctal(offset, length); - } - - int computeUnsignedHeaderChecksum() { - // Accessing the last element first helps the VM eliminate bounds checks in - // the loops below. - this[blockSize - 1]; // ignore: unnecessary_statements - var result = checksumLength * _checksumPlaceholder; - - for (var i = 0; i < checksumOffset; i++) { - result += this[i]; - } - for (var i = _checksumEnd; i < blockSize; i++) { - result += this[i]; - } - - return result; - } - - int computeSignedHeaderChecksum() { - this[blockSize - 1]; // ignore: unnecessary_statements - // Note that _checksumPlaceholder.toSigned(8) == _checksumPlaceholder - var result = checksumLength * _checksumPlaceholder; - - for (var i = 0; i < checksumOffset; i++) { - result += this[i].toSigned(8); - } - for (var i = _checksumEnd; i < blockSize; i++) { - result += this[i].toSigned(8); - } - - return result; - } - - bool matchesHeader(List header, {int offset = magicOffset}) { - for (var i = 0; i < header.length; i++) { - if (this[offset + i] != header[i]) return false; - } - - return true; - } - - bool get isAllZeroes { - for (var i = 0; i < length; i++) { - if (this[i] != 0) return false; - } - - return true; - } -} - -bool isNotAscii(int i) => i > 128; - -/// Like [int.parse], but throwing a [TarException] instead of the more-general -/// [FormatException] when it fails. -int parseInt(String source) { - return int.tryParse(source, radix: 10) ?? - (throw TarException('Not an int: $source')); -} - -/// Takes a [paxTimeString] of the form %d.%d as described in the PAX -/// specification. Note that this implementation allows for negative timestamps, -/// which is allowed for by the PAX specification, but not always portable. -/// -/// Note that Dart's [DateTime] class only allows us to give up to microsecond -/// precision, which implies that we cannot parse all the digits in since PAX -/// allows for nanosecond level encoding. -DateTime parsePaxTime(String paxTimeString) { - const maxMicroSecondDigits = 6; - - /// Split [paxTimeString] into seconds and sub-seconds parts. - var secondsString = paxTimeString; - var microSecondsString = ''; - final position = paxTimeString.indexOf('.'); - if (position >= 0) { - secondsString = paxTimeString.substring(0, position); - microSecondsString = paxTimeString.substring(position + 1); - } - - /// Parse the seconds. - final seconds = int.tryParse(secondsString); - if (seconds == null) { - throw TarException.header('Invalid PAX time $paxTimeString detected!'); - } - - if (microSecondsString.replaceAll(RegExp('[0-9]'), '') != '') { - throw TarException.header( - 'Invalid nanoseconds $microSecondsString detected'); - } - - microSecondsString = microSecondsString.padRight(maxMicroSecondDigits, '0'); - microSecondsString = microSecondsString.substring(0, maxMicroSecondDigits); - - var microSeconds = - microSecondsString.isEmpty ? 0 : int.parse(microSecondsString); - if (paxTimeString.startsWith('-')) microSeconds = -microSeconds; - - return microsecondsSinceEpoch(microSeconds + seconds * pow(10, 6).toInt()); -} - -DateTime secondsSinceEpoch(int timestamp) { - return DateTime.fromMillisecondsSinceEpoch(timestamp * 1000, isUtc: true); -} - -DateTime millisecondsSinceEpoch(int milliseconds) { - return DateTime.fromMillisecondsSinceEpoch(milliseconds, isUtc: true); -} - -DateTime microsecondsSinceEpoch(int microseconds) { - return DateTime.fromMicrosecondsSinceEpoch(microseconds, isUtc: true); -} - -int numBlocks(int fileSize) { - if (fileSize % blockSize == 0) return fileSize ~/ blockSize; - - return fileSize ~/ blockSize + 1; -} - -int nextBlockSize(int fileSize) => numBlocks(fileSize) * blockSize; - -extension ToTyped on List { - Uint8List asUint8List() { - // Flow analysis doesn't work on this. - final $this = this; - return $this is Uint8List ? $this : Uint8List.fromList(this); - } -} - -/// Generates a chunked stream of [length] zeroes. -Stream> zeroes(int length) async* { - // Emit data in chunks for efficiency - const chunkSize = 4 * 1024; - if (length < chunkSize) { - yield Uint8List(length); - return; - } - - final chunk = Uint8List(chunkSize); - for (var i = 0; i < length ~/ chunkSize; i++) { - yield chunk; - } - - final remainingBytes = length % chunkSize; - if (remainingBytes != 0) { - yield Uint8List(remainingBytes); - } -} - -/// An optimized reader reading 512-byte blocks from an input stream. -final class BlockReader { - final Stream> _input; - StreamSubscription>? _subscription; - bool _isClosed = false; - - /// If a request is active, returns the current stream that we're reporting. - /// This controler is synchronous. - StreamController? _outgoing; - - /// The amount of (512-byte) blocks remaining before [_outgoing] should close. - int _remainingBlocksInOutgoing = 0; - - /// A pending tar block that has not been emitted yet. - /// - /// This can happen if we receive small chunks of data in [_onData] that - /// aren't enough to form a full block. - final Uint8List _pendingBlock = Uint8List(blockSize); - - /// The offset in [_pendingBlock] at which new data should start. - /// - /// For instance, if this value is `502`, we're missing `10` additional bytes - /// to complete the [_pendingBlock]. - /// When this value is `0`, there is no active pending block. - int _offsetInPendingBlock = 0; - - /// Additional data that we received, but were unable to dispatch to a - /// downstream listener yet. - /// - /// This can happen if a we receive a large chunk of data and a listener is - /// only interested in a small chunk. - /// - /// We will never have trailing data and a pending block at the same time. - /// When we haver fewer than 512 bytes of trailing data, it should be stored - /// as a pending block instead. - Uint8List? _trailingData; - - /// The offset in the [_trailingData] byte array. - /// - /// When a new listener attaches, we can start by emitting the sublist - /// starting at this offset. - int _offsetInTrailingData = 0; - - BlockReader(this._input); - - /// Emits full blocks. - /// - /// Returns `true` if the listener detached in response to emitting these - /// blocks. In this case, remaining data must be saved in [_trailingData]. - bool _emitBlocks(Uint8List data, {int amount = 1}) { - assert(_remainingBlocksInOutgoing >= amount); - final outgoing = _outgoing!; - - if (!outgoing.isClosed) outgoing.add(data); - - final remainingNow = _remainingBlocksInOutgoing -= amount; - if (remainingNow == 0) { - _outgoing = null; - _pause(); - - // Scheduling this in a microtask becuase the stream controller is - // synchronous. - scheduleMicrotask(() { - // We don't need to await this since the stream controller is not used - // afterwards, if there's a paused listener we don't really care about - // that. - unawaited(outgoing.close()); - }); - return true; - } else if (outgoing.isPaused || outgoing.isClosed) { - _pause(); - return true; - } - - return false; - } - - void _onData(List data) { - assert(_outgoing != null && _trailingData == null); - - final typedData = data.asUint8List(); - var offsetInData = 0; - - /// Saves parts of the current chunks that couldn't be emitted. - void saveTrailingState() { - assert(_trailingData == null && _offsetInPendingBlock == 0); - - final remaining = typedData.length - offsetInData; - - if (remaining == 0) { - return; // Nothing to save, the chunk has been consumed fully. - } else if (remaining < blockSize) { - // Store remaining data as a pending block. - _pendingBlock.setAll(0, typedData.sublistView(offsetInData)); - _offsetInPendingBlock = remaining; - } else { - _trailingData = typedData; - _offsetInTrailingData = offsetInData; - } - } - - // Try to complete a pending block first - var offsetInPending = _offsetInPendingBlock; - final canWriteIntoPending = min(blockSize - offsetInPending, data.length); - - if (offsetInPending != 0 && canWriteIntoPending > 0) { - _pendingBlock.setAll( - offsetInPending, typedData.sublistView(0, canWriteIntoPending)); - offsetInPending = _offsetInPendingBlock += canWriteIntoPending; - offsetInData += canWriteIntoPending; - - // Did this finish the pending block? - if (offsetInPending == blockSize) { - _offsetInPendingBlock = 0; - if (_emitBlocks(_pendingBlock)) { - // Emitting the pending block completed all pending requests. - saveTrailingState(); - return; - } - } else { - // The chunk we received didn't fill up the pending block, so just stop - // here. - assert(offsetInData == data.length); - return; - } - } - - // At this point, the pending block should have been served. - assert(_offsetInPendingBlock == 0); - - final fullBlocksToEmit = min(_remainingBlocksInOutgoing, - (typedData.length - offsetInData) ~/ blockSize); - - if (fullBlocksToEmit > 0) { - _emitBlocks( - typedData.sublistView( - offsetInData, offsetInData += fullBlocksToEmit * blockSize), - amount: fullBlocksToEmit, - ); - } - - saveTrailingState(); - } - - void _onError(Object error, StackTrace trace) { - assert(_outgoing != null && _trailingData == null); - - _outgoing!.addError(error, trace); - } - - void _onDone() { - assert(_outgoing != null && _trailingData == null); - final outgoing = _outgoing!; - - // Add pending data, then close - if (_offsetInPendingBlock != 0) { - outgoing.add(_pendingBlock.sublistView(0, _offsetInPendingBlock)); - } - - _isClosed = true; - - // Can be unawated because this is an onDone callback of the subscription, - // the subscription is already complete and we're just cleaning up. - unawaited(_subscription?.cancel()); - - // Can be unawated because we're fully done here, we won't do anything else - // with the outgoing controller. - unawaited(outgoing.close()); - } - - void _subscribeOrResume() { - // We should not resume the subscription if there is trailing data ready to - // be emitted. - assert(_trailingData == null); - - final sub = _subscription; - if (sub == null) { - _subscription = _input.listen(_onData, - onError: _onError, onDone: _onDone, cancelOnError: true); - } else { - sub.resume(); - } - } - - void _pause() { - final sub = _subscription!; // ignore: cancel_subscriptions - - if (!sub.isPaused) sub.pause(); - } - - Future nextBlock() { - final result = Uint8List(blockSize); - var offset = 0; - - return nextBlocks(1).forEach((chunk) { - result.setAll(offset, chunk); - offset += chunk.length; - }).then((void _) => result.sublistView(0, offset)); - } - - Stream nextBlocks(int amount) { - if (_isClosed || amount == 0) { - return const Stream.empty(); - } - if (_outgoing != null) { - throw StateError( - 'Cannot call nextBlocks() before the previous stream completed.'); - } - assert(_remainingBlocksInOutgoing == 0); - - // We're making this synchronous because we will mostly add events in - // response to receiving chunks from the source stream. We manually ensure - // that other emits are happening asynchronously. - final controller = StreamController(sync: true); - _outgoing = controller; - _remainingBlocksInOutgoing = amount; - - var state = _StreamState.initial; - - /// Sends trailing data to the stream. Returns true if the subscription - /// should still be resumed afterwards. - bool emitTrailing() { - // Attempt to serve requests from pending data first. - final trailing = _trailingData; - if (trailing != null) { - // There should never be trailing data and a pending block at the - // same time - assert(_offsetInPendingBlock == 0); - - var remaining = trailing.length - _offsetInTrailingData; - // If there is trailing data, it should contain a full block - // (otherwise we would have stored it as a pending block) - assert(remaining >= blockSize); - - final blocks = min(_remainingBlocksInOutgoing, remaining ~/ blockSize); - assert(blocks > 0); - - final done = _emitBlocks( - trailing.sublistView(_offsetInTrailingData, - _offsetInTrailingData + blocks * blockSize), - amount: blocks); - - remaining -= blocks * blockSize; - _offsetInTrailingData += blocks * blockSize; - - if (remaining == 0) { - _trailingData = null; - _offsetInTrailingData = 0; - } else if (remaining < blockSize) { - assert(_offsetInPendingBlock == 0); - - // Move trailing data into a pending block - _pendingBlock.setAll(0, trailing.sublistView(_offsetInTrailingData)); - _offsetInPendingBlock = remaining; - _trailingData = null; - _offsetInTrailingData = 0; - } else { - // If there is still more than a full block of data waiting, we - // should not listen. This implies that the stream is done already. - assert(done); - } - - // The listener detached in response to receiving the event. - if (done) { - if (_remainingBlocksInOutgoing == 0) state = _StreamState.done; - return false; - } - } - - return true; - } - - void scheduleInitialEmit() { - scheduleMicrotask(() { - if (state != _StreamState.initial) return; - state = _StreamState.attached; - - if (emitTrailing()) { - _subscribeOrResume(); - } - }); - } - - controller - ..onListen = scheduleInitialEmit - ..onPause = () { - assert( - state == _StreamState.initial || - state == _StreamState.attached || - state == _StreamState.done, - 'Unexpected pause event in $state ($_remainingBlocksInOutgoing blocks remaining).'); - - if (state == _StreamState.initial) { - state = _StreamState.pausedAfterInitial; - } else if (state == _StreamState.attached) { - _pause(); - state = _StreamState.pausedAfterAttached; - } else if (state == _StreamState.done) { - // It may happen that onPause is called in a state where we believe - // the stream to be done already. After the stream is done, we close - // the controller in a new microtask. So if the subscription is paused - // after the last event it emitted but before we close the controller, - // we can get a pause event here. - // There's nothing to do in that case. - assert(_subscription?.isPaused != false); - } - } - ..onResume = () { - // We're done already - if (_remainingBlocksInOutgoing == 0) return; - - assert(state == _StreamState.pausedAfterAttached || - state == _StreamState.pausedAfterInitial); - - if (state == _StreamState.pausedAfterInitial) { - state = _StreamState.initial; - scheduleInitialEmit(); - } else { - state = _StreamState.attached; - if (emitTrailing()) { - _subscribeOrResume(); - } - } - } - ..onCancel = () { - state = _StreamState.done; - }; - - return controller.stream; - } - - FutureOr close() { - _isClosed = true; - return _subscription?.cancel(); - } -} - -enum _StreamState { - initial, - attached, - pausedAfterInitial, - pausedAfterAttached, - done, -} diff --git a/lib/src/third_party/tar/lib/src/writer.dart b/lib/src/third_party/tar/lib/src/writer.dart deleted file mode 100644 index 24f919083..000000000 --- a/lib/src/third_party/tar/lib/src/writer.dart +++ /dev/null @@ -1,499 +0,0 @@ -import 'dart:async'; -import 'dart:convert'; -import 'dart:typed_data'; - -import 'charcodes.dart'; -import 'constants.dart'; -import 'entry.dart'; -import 'format.dart'; -import 'header.dart'; -import 'utils.dart'; - -final class _WritingTransformer - extends StreamTransformerBase> { - final OutputFormat format; - - const _WritingTransformer(this.format); - - @override - Stream> bind(Stream stream) { - // sync because the controller proxies another stream - final controller = StreamController>(sync: true); - controller.onListen = () { - // Can be unawaited since it's the only thing done in onListen and since - // pipe is a terminal operation managing the remaining lifecycle of this - // stream controller. - unawaited(stream.pipe(tarWritingSink(controller, format: format))); - }; - - return controller.stream; - } -} - -/// A stream transformer writing tar entries as byte streams. -/// -/// Regardless of the input stream, the stream returned by this -/// [StreamTransformer.bind] is a single-subscription stream. -/// Apart from that, subscriptions, cancellations, pauses and resumes are -/// propagated as one would expect from a [StreamTransformer]. -/// -/// When piping the resulting stream into a [StreamConsumer], consider using -/// [tarWritingSink] directly. -/// To change the output format of files with long names, use [tarWriterWith]. -const StreamTransformer> tarWriter = - _WritingTransformer(OutputFormat.pax); - -/// Creates a stream transformer writing tar entries as byte streams, with -/// custom encoding options. -/// -/// The [format] [OutputFormat] can be used to select the way tar entries with -/// long file or link names are written. By default, the writer will emit an -/// extended PAX header for the file ([OutputFormat.pax]). -/// Alternatively, [OutputFormat.gnuLongName] can be used to emit special tar -/// entries with the [TypeFlag.gnuLongName] type. -/// -/// Regardless of the input stream, the stream returned by this -/// [StreamTransformer.bind] is a single-subscription stream. -/// Apart from that, subscriptions, cancellations, pauses and resumes are -/// propagated as one would expect from a [StreamTransformer]. -/// -/// When using the default options, prefer using the constant [tarWriter] -/// instead. -StreamTransformer> tarWriterWith( - {OutputFormat format = OutputFormat.pax}) { - return _WritingTransformer(format); -} - -/// Create a sink emitting encoded tar files to the [output] sink. -/// -/// For instance, you can use this to write a tar file: -/// -/// ```dart -/// import 'dart:convert'; -/// import 'dart:io'; -/// import 'package:tar/tar.dart'; -/// -/// Future main() async { -/// Stream entries = Stream.value( -/// TarEntry.data( -/// TarHeader( -/// name: 'example.txt', -/// mode: int.parse('644', radix: 8), -/// ), -/// utf8.encode('This is the content of the tar file'), -/// ), -/// ); -/// -/// final output = File('/tmp/test.tar').openWrite(); -/// await entries.pipe(tarWritingSink(output)); -/// } -/// ``` -/// -/// Note that, if you don't set the [TarHeader.size], outgoing tar entries need -/// to be buffered once, which decreases performance. -/// -/// The [format] argument can be used to control how long file names are written -/// in the tar archive. For more details, see the options in [OutputFormat]. -/// -/// See also: -/// - [tarWriter], a stream transformer using this sink -/// - [StreamSink] -StreamSink tarWritingSink(StreamSink> output, - {OutputFormat format = OutputFormat.pax}) { - return _WritingSink(output, format); -} - -/// A synchronous encoder for in-memory tar files. -/// -/// The default [tarWriter] creates an asynchronous conversion from a stream of -/// tar entries to a byte stream. -/// When all tar entries are in-memory ([SynchronousTarEntry]), it is possible -/// to write them synchronously too. -/// -/// To create a tar archive consisting of a single entry, use -/// [Converter.convert] on this [tarConverter]. -/// To create a tar archive consisting of any number of entries, first call -/// [Converter.startChunkedConversion] with a suitable output sink. Next, call -/// [Sink.add] for each tar entry and finish the archive by calling -/// [Sink.close]. -/// -/// To change the output format of the tar converter, use [tarConverterWith]. -/// To encode any kind of tar entries, use the asynchronous [tarWriter]. -const Converter> tarConverter = - _SynchronousTarConverter(OutputFormat.pax); - -/// A synchronous encoder for in-memory tar files, with custom encoding options. -/// -/// For more information on how to use the converter, see [tarConverter]. -Converter> tarConverterWith( - {OutputFormat format = OutputFormat.pax}) { - return _SynchronousTarConverter(format); -} - -/// This option controls how long file and link names should be written. -/// -/// This option can be passed to writer in [tarWritingSink] or[tarWriterWith]. -enum OutputFormat { - /// Generates an extended PAX headers to encode files with a long name. - /// - /// This is the default option. - pax, - - /// Generates [TypeFlag.gnuLongName] or [TypeFlag.gnuLongLink] entries when - /// encoding files with a long name. - /// - /// When this option is set, `package:tar` will not emit PAX headers which - /// may improve compatibility with some legacy systems like old 7zip versions. - /// - /// Note that this format can't encode large file sizes or long user names. - /// Tar entries can't be written if - /// * their [TarHeader.userName] is longer than 31 bytes in utf8, - /// * their [TarHeader.groupName] is longer than 31 bytes in utf8, or, - /// * their [TarEntry.contents] are larger than 8589934591 byte (around - /// 8 GiB). - /// - /// Attempting to encode such file will throw an [UnsupportedError]. - gnuLongName, -} - -final class _WritingSink implements StreamSink { - final StreamSink> _output; - final _SynchronousTarSink _synchronousWriter; - bool _closed = false; - final Completer _done = Completer(); - - int _pendingOperations = 0; - Future _ready = Future.value(); - - _WritingSink(this._output, OutputFormat format) - : _synchronousWriter = _SynchronousTarSink(_output, format); - - @override - Future get done => _done.future; - - @override - Future add(TarEntry event) { - if (_closed) { - throw StateError('Cannot add event after close was called'); - } - return _doWork(() => _safeAdd(event)); - } - - Future _doWork(FutureOr Function() work) { - _pendingOperations++; - // Chain futures to make sure we only write one entry at a time. - return _ready = _ready - .then((_) => work()) - .catchError(_output.addError) - .whenComplete(() { - _pendingOperations--; - - if (_closed && _pendingOperations == 0) { - _done.complete(_output.close()); - } - }); - } - - Future _safeAdd(TarEntry event) async { - final header = event.header; - var size = header.size; - Uint8List? bufferedData; - if (size < 0) { - final builder = BytesBuilder(); - await event.contents.forEach(builder.add); - bufferedData = builder.takeBytes(); - size = bufferedData.length; - } - - _synchronousWriter._writeHeader(header, size); - - // Write content. - if (bufferedData != null) { - _output.add(bufferedData); - } else { - await _output.addStream(event.contents); - } - - _output.add(_paddingBytes(size)); - } - - @override - void addError(Object error, [StackTrace? stackTrace]) { - _output.addError(error, stackTrace); - } - - @override - Future addStream(Stream stream) async { - await for (final entry in stream) { - await add(entry); - } - } - - @override - Future close() async { - if (!_closed) { - _closed = true; - - // Add two empty blocks at the end. - await _doWork(_synchronousWriter.close); - } - - return done; - } -} - -Uint8List _paddingBytes(int size) { - final padding = -size % blockSize; - assert((size + padding) % blockSize == 0 && - padding <= blockSize && - padding >= 0); - - return Uint8List(padding); -} - -final class _SynchronousTarConverter - extends Converter> { - final OutputFormat format; - - const _SynchronousTarConverter(this.format); - - @override - Sink startChunkedConversion(Sink> sink) { - return _SynchronousTarSink(sink, format); - } - - @override - List convert(SynchronousTarEntry input) { - final output = BytesBuilder(copy: false); - startChunkedConversion(ByteConversionSink.withCallback(output.add)) - ..add(input) - ..close(); - - return output.takeBytes(); - } -} - -final class _SynchronousTarSink implements Sink { - final OutputFormat _format; - final Sink> _output; - - bool _closed = false; - int _paxHeaderCount = 0; - - _SynchronousTarSink(this._output, this._format); - - @override - void add(SynchronousTarEntry data) { - addHeaderAndData(data.header, data.data); - } - - void addHeaderAndData(TarHeader header, List data) { - _throwIfClosed(); - - _writeHeader(header, data.length); - _output - ..add(data) - ..add(_paddingBytes(data.length)); - } - - @override - void close() { - if (_closed) return; - - // End the tar archive by writing two zero blocks. - _output - ..add(UnmodifiableUint8ListView(zeroBlock)) - ..add(UnmodifiableUint8ListView(zeroBlock)); - _output.close(); - - _closed = true; - } - - void _throwIfClosed() { - if (_closed) { - throw StateError('Encoder is closed. ' - 'After calling `endOfArchive()`, encoder must not be used.'); - } - } - - void _writeHeader(TarHeader header, int size) { - assert(header.size < 0 || header.size == size); - - var nameBytes = utf8.encode(header.name); - var linkBytes = utf8.encode(header.linkName ?? ''); - var gnameBytes = utf8.encode(header.groupName ?? ''); - var unameBytes = utf8.encode(header.userName ?? ''); - - // We only get 100 chars for the name and link name. If they are longer, we - // have to insert an entry just to store the names. Some tar implementations - // expect them to be zero-terminated, so use 99 chars to be safe. - final paxHeader = >{}; - - if (nameBytes.length > 99) { - paxHeader[paxPath] = nameBytes; - nameBytes = nameBytes.sublist(0, 99); - } - if (linkBytes.length > 99) { - paxHeader[paxLinkpath] = linkBytes; - linkBytes = linkBytes.sublist(0, 99); - } - - // It's even worse for users and groups, where we only get 31 usable chars. - if (gnameBytes.length > 31) { - paxHeader[paxGname] = gnameBytes; - gnameBytes = gnameBytes.sublist(0, 31); - } - if (unameBytes.length > 31) { - paxHeader[paxUname] = unameBytes; - unameBytes = unameBytes.sublist(0, 31); - } - - if (size > maxIntFor12CharOct) { - paxHeader[paxSize] = ascii.encode(size.toString()); - } - - if (paxHeader.isNotEmpty) { - if (_format == OutputFormat.pax) { - _writePaxHeader(paxHeader); - } else { - _writeGnuLongName(paxHeader); - } - } - - final headerBlock = Uint8List(blockSize) - ..setAll(0, nameBytes) - ..setUint(header.mode, 100, 8) - ..setUint(header.userId, 108, 8) - ..setUint(header.groupId, 116, 8) - ..setUint(size, 124, 12) - ..setUint(header.modified.millisecondsSinceEpoch ~/ 1000, 136, 12) - ..[156] = typeflagToByte(header.typeFlag) - ..setAll(157, linkBytes) - ..setAll(257, magicUstar) - ..setUint(0, 263, 2) // version - ..setAll(265, unameBytes) - ..setAll(297, gnameBytes) - // To calculate the checksum, we first fill the checksum range with spaces - ..setAll(148, List.filled(8, $space)); - - // Then, we take the sum of the header - var checksum = 0; - for (final byte in headerBlock) { - checksum += byte; - } - headerBlock.setUint(checksum, 148, 8); - _output.add(headerBlock); - } - - /// Encodes an extended pax header. - /// - /// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_03 - void _writePaxHeader(Map> values) { - final buffer = BytesBuilder(); - // format of each entry: "%d %s=%s\n", , , - // note that the length includes the trailing \n and the length description - // itself. - values.forEach((key, value) { - final encodedKey = utf8.encode(key); - // +3 for the whitespace, the equals and the \n - final payloadLength = encodedKey.length + value.length + 3; - var indicatedLength = payloadLength; - - // The indicated length contains the length (in decimals) itself. So if - // we had payloadLength=9, then we'd prefix a 9 at which point the whole - // string would have a length of 10. If that happens, increment length. - var actualLength = payloadLength + indicatedLength.toString().length; - - while (actualLength != indicatedLength) { - indicatedLength++; - actualLength = payloadLength + indicatedLength.toString().length; - } - - // With that sorted out, let's add the line - buffer - ..add(utf8.encode(indicatedLength.toString())) - ..addByte($space) - ..add(encodedKey) - ..addByte($equal) - ..add(value) - ..addByte($lf); // \n - }); - - final paxData = buffer.takeBytes(); - addHeaderAndData( - HeaderImpl.internal( - format: TarFormat.pax, - modified: millisecondsSinceEpoch(0), - name: 'PaxHeader/${_paxHeaderCount++}', - mode: 0, - size: paxData.length, - typeFlag: TypeFlag.xHeader, - ), - paxData, - ); - } - - void _writeGnuLongName(Map> values) { - // Ensure that a file that can't be written in the GNU format is not written - const allowedKeys = {paxPath, paxLinkpath}; - final invalidOptions = values.keys.toSet()..removeAll(allowedKeys); - if (invalidOptions.isNotEmpty) { - throw UnsupportedError( - 'Unsupporteed entry for OutputFormat.gnu. It uses long fields that ' - "can't be represented: $invalidOptions. \n" - 'Try using OutputFormat.pax instead.', - ); - } - - final name = values[paxPath]; - final linkName = values[paxLinkpath]; - - void create(List name, TypeFlag flag) { - return addHeaderAndData( - HeaderImpl.internal( - name: '././@LongLink', - modified: millisecondsSinceEpoch(0), - format: TarFormat.gnu, - typeFlag: flag, - ), - name, - ); - } - - if (name != null) { - create(name, TypeFlag.gnuLongName); - } - if (linkName != null) { - create(linkName, TypeFlag.gnuLongLink); - } - } -} - -extension on Uint8List { - void setUint(int value, int position, int length) { - // Values are encoded as octal string, terminated and left-padded with - // space chars. - - // Set terminating space char. - this[position + length - 1] = $space; - - // Write as octal value, we write from right to left - var number = value; - var needsExplicitZero = number == 0; - - for (var pos = position + length - 2; pos >= position; pos--) { - if (number != 0) { - // Write the last octal digit of the number (e.g. the last 4 bits) - this[pos] = (number & 7) + $0; - // then drop the last digit (divide by 8 = 2³) - number >>= 3; - } else if (needsExplicitZero) { - this[pos] = $0; - needsExplicitZero = false; - } else { - // done, left-pad with spaces - this[pos] = $space; - } - } - } -} diff --git a/lib/src/third_party/tar/lib/tar.dart b/lib/src/third_party/tar/lib/tar.dart deleted file mode 100644 index 9948c4f71..000000000 --- a/lib/src/third_party/tar/lib/tar.dart +++ /dev/null @@ -1,16 +0,0 @@ -/// Streaming tar implementation for Dart. -/// -/// To read tar files, see [TarReader]. To write tar files, use [tarWritingSink] -/// or [tarWriter]. -library tar; - -// For dartdoc. -import 'src/reader.dart'; -import 'src/writer.dart'; - -export 'src/entry.dart' show TarEntry, SynchronousTarEntry; -export 'src/exception.dart'; -export 'src/format.dart'; -export 'src/header.dart' show TarHeader, TypeFlag; -export 'src/reader.dart' show TarReader; -export 'src/writer.dart'; diff --git a/lib/src/third_party/tar/vendored-pubspec.yaml b/lib/src/third_party/tar/vendored-pubspec.yaml deleted file mode 100644 index 5d641aa14..000000000 --- a/lib/src/third_party/tar/vendored-pubspec.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: tar -description: Memory-efficient, streaming implementation of the tar file format -version: 1.0.1 -repository: https://github.com/simolus3/tar/ - -environment: - sdk: '>=3.0.0 <4.0.0' - -dependencies: - async: ^2.6.0 - meta: ^1.3.0 - typed_data: ^1.3.0 - -dev_dependencies: - charcode: ^1.2.0 - extra_pedantic: ^4.0.0 - file: ^6.1.2 - node_io: ^2.1.0 - path: ^1.8.0 - test: ^1.20.0 diff --git a/lib/src/third_party/vendor-state.yaml b/lib/src/third_party/vendor-state.yaml deleted file mode 100644 index 86c5b42e4..000000000 --- a/lib/src/third_party/vendor-state.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# DO NOT EDIT: This file is generated by package:vendor version 0.9.0 -version: 0.9.0 -config: - import_rewrites: - tar: tar - vendored_dependencies: - tar: - package: tar - version: 1.0.1 - import_rewrites: {} - include: - - pubspec.yaml - - README.md - - LICENSE - - CHANGELOG.md - - lib/** - - analysis_options.yaml diff --git a/pubspec.yaml b/pubspec.yaml index a9d1321d2..bce7a4bea 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -25,6 +25,7 @@ dependencies: shelf: ^1.1.1 source_span: ^1.8.1 stack_trace: ^1.10.0 + tar: ^1.0.1 typed_data: ^1.3.1 usage: ^4.0.2 yaml: ^3.1.0 @@ -36,4 +37,3 @@ dev_dependencies: test: ^1.21.5 test_descriptor: ^2.0.0 test_process: ^2.0.0 - vendor: ^0.9.5 diff --git a/test/io_test.dart b/test/io_test.dart index d788e139f..83ba9c208 100644 --- a/test/io_test.dart +++ b/test/io_test.dart @@ -9,7 +9,7 @@ import 'dart:io'; import 'package:path/path.dart' as path; import 'package:pub/src/exceptions.dart'; import 'package:pub/src/io.dart'; -import 'package:pub/src/third_party/tar/lib/tar.dart'; +import 'package:tar/tar.dart'; import 'package:test/test.dart'; import 'descriptor.dart' as d; diff --git a/test/test_pub.dart b/test/test_pub.dart index 395a99ebd..951ee21b1 100644 --- a/test/test_pub.dart +++ b/test/test_pub.dart @@ -29,10 +29,10 @@ import 'package:pub/src/log.dart' as log; import 'package:pub/src/package_name.dart'; import 'package:pub/src/source/hosted.dart'; import 'package:pub/src/system_cache.dart'; -import 'package:pub/src/third_party/tar/lib/tar.dart'; import 'package:pub/src/utils.dart'; import 'package:pub/src/validator.dart'; import 'package:pub_semver/pub_semver.dart'; +import 'package:tar/tar.dart'; import 'package:test/test.dart' hide fail; import 'package:test/test.dart' as test show fail; import 'package:test_process/test_process.dart'; diff --git a/vendor.yaml b/vendor.yaml deleted file mode 100644 index 287f06f81..000000000 --- a/vendor.yaml +++ /dev/null @@ -1,6 +0,0 @@ -import_rewrites: - tar: tar -vendored_dependencies: - tar: - package: tar - version: 1.0.1