diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..3253e2c --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,16 @@ +name: "CodeQL" + +on: [pull_request] +jobs: + lint: + name: CodeQL + runs-on: ubuntu-latest + + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Run CodeQL + run: | + docker run --rm -v $PWD:/app composer sh -c \ + "composer install --profile --ignore-platform-reqs && composer check" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 00096f3..ff12c67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,9 +20,13 @@ jobs: - name: Validate composer.json and composer.lock run: composer validate --strict + + - name: Install dependencies + run: composer install --ignore-platform-reqs --optimize-autoloader --no-plugins --no-scripts --prefer-dist - - name: Compose install - run: composer install --ignore-platform-reqs + - name: Start container + # For local testing, also run this before retrying tests: docker rm --force $(docker ps -aq) + run: docker compose up -d && sleep 15 - name: Run tests - run: composer test \ No newline at end of file + run: docker compose exec tests vendor/bin/phpunit --configuration phpunit.xml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 5e8a951..e913320 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,16 +9,22 @@ COPY composer.lock /usr/local/src/ COPY composer.json /usr/local/src/ RUN composer install --ignore-platform-reqs --optimize-autoloader --no-plugins --no-scripts --prefer-dist - + +RUN docker-php-ext-install sockets + FROM php:8.0-cli-alpine as final +ENV DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} +ENV DOCKER_API_VERSION=1.43 + LABEL maintainer="team@appwrite.io" RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN \ apk update \ - && apk add --no-cache make automake autoconf gcc g++ git brotli-dev \ + && apk add --no-cache make automake autoconf gcc g++ git brotli-dev docker-cli \ + && docker-php-ext-install sockets \ && docker-php-ext-install opcache WORKDIR /usr/src/code diff --git a/composer.lock b/composer.lock index 93e43eb..44535d1 100644 --- a/composer.lock +++ b/composer.lock @@ -178,16 +178,16 @@ }, { "name": "laravel/pint", - "version": "v1.17.0", + "version": "v1.17.2", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "4dba80c1de4b81dc4c4fb10ea6f4781495eb29f5" + "reference": "e8a88130a25e3f9d4d5785e6a1afca98268ab110" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/4dba80c1de4b81dc4c4fb10ea6f4781495eb29f5", - "reference": "4dba80c1de4b81dc4c4fb10ea6f4781495eb29f5", + "url": "https://api.github.com/repos/laravel/pint/zipball/e8a88130a25e3f9d4d5785e6a1afca98268ab110", + "reference": "e8a88130a25e3f9d4d5785e6a1afca98268ab110", "shasum": "" }, "require": { @@ -198,13 +198,13 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.59.3", - "illuminate/view": "^10.48.12", - "larastan/larastan": "^2.9.7", + "friendsofphp/php-cs-fixer": "^3.61.1", + "illuminate/view": "^10.48.18", + "larastan/larastan": "^2.9.8", "laravel-zero/framework": "^10.4.0", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.15.1", - "pestphp/pest": "^2.34.8" + "pestphp/pest": "^2.35.0" }, "bin": [ "builds/pint" @@ -240,7 +240,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-07-23T16:40:20+00:00" + "time": "2024-08-06T15:11:54+00:00" }, { "name": "myclabs/deep-copy", @@ -480,16 +480,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.11.8", + "version": "1.11.10", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "6adbd118e6c0515dd2f36b06cde1d6da40f1b8ec" + "reference": "640410b32995914bde3eed26fa89552f9c2c082f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/6adbd118e6c0515dd2f36b06cde1d6da40f1b8ec", - "reference": "6adbd118e6c0515dd2f36b06cde1d6da40f1b8ec", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/640410b32995914bde3eed26fa89552f9c2c082f", + "reference": "640410b32995914bde3eed26fa89552f9c2c082f", "shasum": "" }, "require": { @@ -534,7 +534,7 @@ "type": "github" } ], - "time": "2024-07-24T07:01:22+00:00" + "time": "2024-08-08T09:02:50+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/docker-compose.yml b/docker-compose.yml index d7c2211..9123116 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,9 @@ services: context: . networks: - orchestration - volumes: + environment: + HOST_DIR: "$PWD" # Nessessary to mount test resources to child containers + volumes: - ./:/usr/src/code - /var/run/docker.sock:/var/run/docker.sock diff --git a/src/Orchestration/Adapter/DockerAPI.php b/src/Orchestration/Adapter/DockerAPI.php index 2b6d785..056af31 100644 --- a/src/Orchestration/Adapter/DockerAPI.php +++ b/src/Orchestration/Adapter/DockerAPI.php @@ -130,34 +130,50 @@ protected function streamCall(string $url, int $timeout = -1): array $stdout = ''; $stderr = ''; - $callback = function (mixed $ch, string $str) use (&$stdout, &$stderr): int { + $isHeader = true; + $currentHeader = null; + $currentData = ''; + + $callback = function (mixed $ch, string $str) use (&$stdout, &$stderr, &$isHeader, &$currentHeader, &$currentData): int { if (empty($str)) { return 0; } - $rawStream = unpack('C*', $str); - $stream = $rawStream[1]; // 1-based index, not 0-based - - // Ascii encoding support - if ($stream === \ord('1')) { - $stream = 1; - } elseif ($stream === \ord('2')) { - $stream = 2; - } - - switch ($stream) { // only 1 or 2, as set while creating exec - case 1: - $packed = pack('C*', ...\array_slice($rawStream, 8)); - $stdout .= $packed; - break; - case 2: - $packed = pack('C*', ...\array_slice($rawStream, 8)); - $stderr .= $packed; - break; + $originalSize = \mb_strlen($str); + + while (! empty($str)) { + if ($isHeader) { + $header = \unpack('Ctype/Cfill1/Cfill2/Cfill3/Nsize', $str); + $str = \mb_strcut($str, 8, null); + $isHeader = false; + $currentHeader = $header; + } else { + $size = $currentHeader['size']; + $type = $currentHeader['type']; + + if (\strlen($str) >= $size) { + $currentData .= \mb_substr($str, 0, $size); + $str = \mb_strcut($str, $size, null); + $isHeader = true; + $currentHeader = null; + + if ($type === 1) { + $stdout .= $currentData; + } else { + $stderr .= $currentData; + } + $currentData = ''; + } else { + $currentHeader['size'] -= \mb_strlen($str); + $currentData .= $str; + $str = ''; + } + } } - return strlen($str); // must return full frame from callback + return $originalSize; // must return full frame from callback }; + \curl_setopt($ch, CURLOPT_WRITEFUNCTION, $callback); \curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); diff --git a/tests/Orchestration/Base.php b/tests/Orchestration/Base.php index b6be5f7..2cac60a 100644 --- a/tests/Orchestration/Base.php +++ b/tests/Orchestration/Base.php @@ -17,9 +17,18 @@ abstract protected static function getAdapterName(): string; */ public static $containerID; - public function setUp(): void {} + public function setUp(): void + { + \exec('rm -rf /usr/src/code/tests/Orchestration/Resources/screens'); // cleanup - public function tearDown(): void {} + \exec('sh -c "cd /usr/src/code/tests/Orchestration/Resources && tar -zcf ./php.tar.gz php"'); + \exec('sh -c "cd /usr/src/code/tests/Orchestration/Resources && tar -zcf ./timeout.tar.gz timeout"'); + } + + public function tearDown(): void + { + \exec('rm -rf /usr/src/code/tests/Orchestration/Resources/screens'); // cleanup + } public function testPullImage(): void { @@ -58,10 +67,10 @@ public function testCreateContainer(): void '', '/usr/local/src/', [ - __DIR__.'/Resources:/test:rw', + \getenv('HOST_DIR').'/tests/Orchestration/Resources:/test:rw', ], [], - __DIR__.'/Resources' + \getenv('HOST_DIR').'/tests/Orchestration/Resources' ); $this->assertNotEmpty($response); @@ -78,22 +87,22 @@ public function testCreateContainer(): void '', '/usr/local/src/', [ - __DIR__.'/Resources:/test:rw', + \getenv('HOST_DIR').'/tests/Orchestration/Resources:/test:rw', ], [], - __DIR__.'/Resources', + \getenv('HOST_DIR').'/tests/Orchestration/Resources', restart: DockerAPI::RESTART_ALWAYS ); $this->assertNotEmpty($response); - sleep(7); + sleep(10); // Docker restart can take quite long to restart. This is safety to prevent flaky tests $output = []; \exec('docker logs '.$response, $output); $output = \implode("\n", $output); $occurances = \substr_count($output, 'Custom start'); - $this->assertGreaterThanOrEqual(5, $occurances); + $this->assertGreaterThanOrEqual(2, $occurances); // 2 logs mean it restarted at least once $response = static::getOrchestration()->remove('TestContainerWithRestart', true); $this->assertEquals(true, $response); @@ -110,10 +119,10 @@ public function testCreateContainer(): void '', '/usr/local/src/', [ - __DIR__.'/Resources:/test:rw', + \getenv('HOST_DIR').'/tests/Orchestration/Resources:/test:rw', ], [], - __DIR__.'/Resources', + \getenv('HOST_DIR').'/tests/Orchestration/Resources', restart: DockerAPI::RESTART_NO ); @@ -147,7 +156,7 @@ public function testCreateContainer(): void '/usr/local/src/', [], [], - __DIR__.'/Resources', + \getenv('HOST_DIR').'/tests/Orchestration/Resources', ); /** @@ -167,7 +176,7 @@ public function testCreateContainer(): void '/usr/local/src/', [], [], - __DIR__.'/Resources', + \getenv('HOST_DIR').'/tests/Orchestration/Resources', ); } @@ -220,7 +229,7 @@ public function testNetworkConnect(): void [ 'teasdsa' => '', ], - __DIR__.'/Resources', + \getenv('HOST_DIR').'/tests/Orchestration/Resources', [ 'test2' => 'Hello World!', ], @@ -268,37 +277,64 @@ public function testExecContainer(): void */ $output = ''; - $this->expectException(\Exception::class); - - static::getOrchestration()->execute( - '60clotVWpufbEpy33zJLcoYHrUTqWaD1FV0FZWsw', // Non-Existent Container - [ - 'php', - 'index.php', - ], - $output - ); + $threwException = false; + try { + static::getOrchestration()->execute( + '60clotVWpufbEpy33zJLcoYHrUTqWaD1FV0FZWsw', // Non-Existent Container + [ + 'php', + 'index.php', + ], + $output + ); + } catch (\Exception $err) { + $threwException = true; + } + $this->assertTrue($threwException); /** * Test for Failure */ $output = ''; - $this->expectException(\Exception::class); + $threwException = false; + try { + static::getOrchestration()->execute( + 'TestContainer', + [ + 'php', + 'doesnotexist.php', // Non-Existent File + ], + $output, + [ + 'test' => 'testEnviromentVariable', + ], + 1 + ); + } catch (\Exception $err) { + $threwException = true; + } + $this->assertTrue($threwException); + + /** + * Test for Success + */ + $output = ''; static::getOrchestration()->execute( 'TestContainer', [ 'php', - 'doesnotexist.php', // Non-Existent File + 'index.php', ], $output, [ 'test' => 'testEnviromentVariable', ], - 1 ); + $this->assertEquals('Hello World! testEnviromentVariable', $output); + /** * Test for Success */ @@ -307,16 +343,20 @@ public function testExecContainer(): void static::getOrchestration()->execute( 'TestContainer', [ - 'php', - 'index.php', - ], - $output, - [ - 'test' => 'testEnviromentVariable', + 'sh', + 'logs.sh', ], + $output ); - $this->assertEquals('Hello World! testEnviromentVariable', $output); + $length = 0; + $length += 1024 * 1024 * 5; // 5MB + $length += 5; // "start" + $length += 3; // "end" + + $this->assertEquals($length, \strlen($output)); + $this->assertStringStartsWith('START', $output); + $this->assertStringEndsWith('END', $output); } /** @@ -358,7 +398,7 @@ public function testTimeoutContainer(): void [ 'teasdsa' => '', ], - __DIR__.'/Resources', + \getenv('HOST_DIR').'/tests/Orchestration/Resources', [ 'test2' => 'Hello World!', ] @@ -372,19 +412,22 @@ public function testTimeoutContainer(): void * Test for Failure */ $output = ''; - - $this->expectException(\Exception::class); - - $response = static::getOrchestration()->execute( - 'TestContainerTimeout', - [ - 'php', - 'index.php', - ], - $output, - [], - 1 - ); + $threwException = false; + try { + $response = static::getOrchestration()->execute( + 'TestContainerTimeout', + [ + 'php', + 'index.php', + ], + $output, + [], + 1 + ); + } catch (\Exception $err) { + $threwException = true; + } + $this->assertTrue($threwException); /** * Test for Success @@ -414,7 +457,7 @@ public function testTimeoutContainer(): void [ 'sh', '-c', - 'echo Hello World!', + 'echo -n Hello World!', // -n prevents from adding linebreak afterwards ], $output, [], @@ -454,7 +497,7 @@ public function testListFilters(): void } /** - * @depends testCreateContainer + * @depends testExecContainer */ public function testRemoveContainer(): void { @@ -530,7 +573,7 @@ public function testRunRemove(): void [ 'teasdsa' => '', ], - __DIR__.'/Resources', + \getenv('HOST_DIR').'/tests/Orchestration/Resources', [ 'test2' => 'Hello World!', ], @@ -557,7 +600,8 @@ public function testUsageStats(): void * Test for Success */ $stats = static::getOrchestration()->getStats(); - $this->assertCount(0, $stats, 'Container(s) still running: '.\json_encode($stats, JSON_PRETTY_PRINT)); + // 1 expected due to container running tests + $this->assertCount(1, $stats, 'Container(s) still running: '.\json_encode($stats, JSON_PRETTY_PRINT)); // This allows CPU-heavy load check static::getOrchestration()->setCpus(1); @@ -571,7 +615,7 @@ public function testUsageStats(): void 'apk update && apk add screen && tail -f /dev/null', ], workdir: '/usr/local/src/', - mountFolder: __DIR__.'/Resources', + mountFolder: \getenv('HOST_DIR').'/tests/Orchestration/Resources', labels: ['utopia-container-type' => 'stats'] ); @@ -586,7 +630,7 @@ public function testUsageStats(): void 'apk update && apk add screen && tail -f /dev/null', ], workdir: '/usr/local/src/', - mountFolder: __DIR__.'/Resources', + mountFolder: \getenv('HOST_DIR').'/tests/Orchestration/Resources', ); $this->assertNotEmpty($containerId2); @@ -603,7 +647,7 @@ public function testUsageStats(): void // Fetch stats, should include high CPU usage $stats = static::getOrchestration()->getStats(); - $this->assertCount(2, $stats); + $this->assertCount(2 + 1, $stats); // +1 due to container running tests $this->assertNotEmpty($stats[0]->getContainerId()); $this->assertEquals(64, \strlen($stats[0]->getContainerId())); diff --git a/tests/Orchestration/Resources/.gitignore b/tests/Orchestration/Resources/.gitignore new file mode 100644 index 0000000..c32b546 --- /dev/null +++ b/tests/Orchestration/Resources/.gitignore @@ -0,0 +1 @@ +*.tar.gz \ No newline at end of file diff --git a/tests/Orchestration/Resources/php.tar.gz b/tests/Orchestration/Resources/php.tar.gz deleted file mode 100644 index a2fccbb..0000000 Binary files a/tests/Orchestration/Resources/php.tar.gz and /dev/null differ diff --git a/tests/Orchestration/Resources/php/logs.sh b/tests/Orchestration/Resources/php/logs.sh new file mode 100644 index 0000000..f8d0e2b --- /dev/null +++ b/tests/Orchestration/Resources/php/logs.sh @@ -0,0 +1,16 @@ +set -e + +CHARS_128="11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" +CHARS_1KB="$CHARS_128$CHARS_128$CHARS_128$CHARS_128$CHARS_128$CHARS_128$CHARS_128$CHARS_128" +CHARS_16KB="$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB$CHARS_1KB" +CHARS_128KB="$CHARS_16KB$CHARS_16KB$CHARS_16KB$CHARS_16KB$CHARS_16KB$CHARS_16KB$CHARS_16KB$CHARS_16KB" +CHARS_1MB="$CHARS_128KB$CHARS_128KB$CHARS_128KB$CHARS_128KB$CHARS_128KB$CHARS_128KB$CHARS_128KB$CHARS_128KB" + +# 5 * CHARS_1MB +echo -n "START" +echo -n "$CHARS_1MB" +echo -n "$CHARS_1MB" +echo -n "$CHARS_1MB" +echo -n "$CHARS_1MB" +echo -n "$CHARS_1MB" +echo -n "END" \ No newline at end of file diff --git a/tests/Orchestration/Resources/timeout.tar.gz b/tests/Orchestration/Resources/timeout.tar.gz deleted file mode 100644 index d1c7c03..0000000 Binary files a/tests/Orchestration/Resources/timeout.tar.gz and /dev/null differ