From 13632321bc2890f0a6d32a23242402bec36120de Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Tue, 3 Sep 2024 08:33:00 +0200 Subject: [PATCH 1/6] Copy Files From External Storage To Local Storage for Import --- src/Snapshot.php | 92 +++++++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 33 deletions(-) diff --git a/src/Snapshot.php b/src/Snapshot.php index dcde77b..86579af 100644 --- a/src/Snapshot.php +++ b/src/Snapshot.php @@ -3,6 +3,7 @@ namespace Spatie\DbSnapshots; use Carbon\Carbon; +use Illuminate\Contracts\Filesystem\Factory; use Illuminate\Filesystem\FilesystemAdapter as Disk; use Illuminate\Support\Facades\DB; use Illuminate\Support\LazyCollection; @@ -10,6 +11,7 @@ use Spatie\DbSnapshots\Events\DeletingSnapshot; use Spatie\DbSnapshots\Events\LoadedSnapshot; use Spatie\DbSnapshots\Events\LoadingSnapshot; +use Spatie\TemporaryDirectory\TemporaryDirectory; class Snapshot { @@ -25,6 +27,8 @@ class Snapshot public const STREAM_BUFFER_SIZE = 16384; + protected Factory $filesystemFactory; + public function __construct(Disk $disk, string $fileName) { $this->disk = $disk; @@ -39,6 +43,8 @@ public function __construct(Disk $disk, string $fileName) } $this->name = pathinfo($fileName, PATHINFO_FILENAME); + + $this->filesystemFactory = app(Factory::class); } public function useStream() @@ -90,45 +96,65 @@ protected function shouldIgnoreLine(string $line): bool protected function loadStream(string $connectionName = null) { - LazyCollection::make(function () { - $stream = $this->compressionExtension === 'gz' - ? gzopen($this->disk->path($this->fileName), 'r') - : $this->disk->readStream($this->fileName); - - $statement = ''; - while (! feof($stream)) { - $chunk = $this->compressionExtension === 'gz' - ? gzread($stream, self::STREAM_BUFFER_SIZE) - : fread($stream, self::STREAM_BUFFER_SIZE); + $directory = (new TemporaryDirectory(config('db-snapshots.temporary_directory_path')))->create(); - $lines = explode("\n", $chunk); - foreach ($lines as $idx => $line) { - if ($this->shouldIgnoreLine($line)) { - continue; - } + config([ + 'filesystems.disks.' . self::class => [ + 'driver' => 'local', + 'root' => $directory->path(), + 'throw' => false, + ] + ]); - $statement .= $line; + $localDisk = $this->filesystemFactory->disk(self::class); - // Carry-over the last line to the next chunk since it - // is possible that this chunk finished mid-line right on - // a semi-colon. - if (count($lines) == $idx + 1) { - break; - } + try { + LazyCollection::make(function () use ($localDisk) { + $localDisk->writeStream($this->fileName, $this->disk->readStream($this->fileName)); + + $stream = $this->compressionExtension === 'gz' + ? gzopen($localDisk->path($this->fileName), 'r') + : $localDisk->readStream($this->fileName); - if (substr(trim($statement), -1, 1) === ';') { - yield $statement; - $statement = ''; + $statement = ''; + while (! feof($stream)) { + $chunk = $this->compressionExtension === 'gz' + ? gzread($stream, self::STREAM_BUFFER_SIZE) + : fread($stream, self::STREAM_BUFFER_SIZE); + + $lines = explode("\n", $chunk); + foreach ($lines as $idx => $line) { + if ($this->shouldIgnoreLine($line)) { + continue; + } + + $statement .= $line; + + // Carry-over the last line to the next chunk since it + // is possible that this chunk finished mid-line right on + // a semi-colon. + if (count($lines) == $idx + 1) { + break; + } + + if (substr(trim($statement), -1, 1) === ';') { + yield $statement; + $statement = ''; + } } } - } - - if (substr(trim($statement), -1, 1) === ';') { - yield $statement; - } - })->each(function (string $statement) use ($connectionName) { - DB::connection($connectionName)->unprepared($statement); - }); + + if ($this->compressionExtension === 'gz') { + gzclose($stream); + } else { + fclose($stream); + } + })->each(function (string $statement) use ($connectionName) { + DB::connection($connectionName)->unprepared($statement); + }); + } finally { + $directory->delete(); + } } public function delete(): void From 9bba682b1f523d741430dcaa0f3e8d47c6ae0d65 Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Mon, 30 Dec 2024 15:50:47 +0100 Subject: [PATCH 2/6] php 8.4 --- src/Snapshot.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Snapshot.php b/src/Snapshot.php index 0855106..ceebb5f 100644 --- a/src/Snapshot.php +++ b/src/Snapshot.php @@ -94,7 +94,7 @@ protected function shouldIgnoreLine(string $line): bool return empty($line) || $this->isASqlComment($line); } - protected function loadStream(string $connectionName = null) + protected function loadStream(?string $connectionName = null): void { $directory = (new TemporaryDirectory(config('db-snapshots.temporary_directory_path')))->create(); @@ -160,7 +160,7 @@ protected function loadStream(string $connectionName = null) $directory->delete(); } } - + public function delete(): void { event(new DeletingSnapshot($this)); From 34b6a76d4516166dadf89a709fee1ccc2ed13235 Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Mon, 30 Dec 2024 16:23:43 +0100 Subject: [PATCH 3/6] Optimize code / add tests --- src/Snapshot.php | 149 ++++++++++++++++++++++-------------- tests/Commands/LoadTest.php | 106 +++++++++++++++++++++++++ 2 files changed, 197 insertions(+), 58 deletions(-) diff --git a/src/Snapshot.php b/src/Snapshot.php index ceebb5f..4f1c596 100644 --- a/src/Snapshot.php +++ b/src/Snapshot.php @@ -4,9 +4,8 @@ use Carbon\Carbon; use Illuminate\Contracts\Filesystem\Factory; -use Illuminate\Filesystem\FilesystemAdapter as Disk; +use Illuminate\Filesystem\FilesystemAdapter; use Illuminate\Support\Facades\DB; -use Illuminate\Support\LazyCollection; use Spatie\DbSnapshots\Events\DeletedSnapshot; use Spatie\DbSnapshots\Events\DeletingSnapshot; use Spatie\DbSnapshots\Events\LoadedSnapshot; @@ -15,7 +14,7 @@ class Snapshot { - public Disk $disk; + public FilesystemAdapter $disk; public string $fileName; @@ -29,10 +28,9 @@ class Snapshot protected Factory $filesystemFactory; - public function __construct(Disk $disk, string $fileName) + public function __construct(FilesystemAdapter $disk, string $fileName) { $this->disk = $disk; - $this->fileName = $fileName; $pathinfo = pathinfo($fileName); @@ -43,14 +41,12 @@ public function __construct(Disk $disk, string $fileName) } $this->name = pathinfo($fileName, PATHINFO_FILENAME); - $this->filesystemFactory = app(Factory::class); } public function useStream(): self { $this->useStream = true; - return $this; } @@ -79,6 +75,10 @@ protected function loadAsync(?string $connectionName = null): void $dbDumpContents = gzdecode($dbDumpContents); } + if (empty(trim($dbDumpContents))) { + return; + } + DB::connection($connectionName)->unprepared($dbDumpContents); } @@ -90,83 +90,116 @@ protected function isASqlComment(string $line): bool protected function shouldIgnoreLine(string $line): bool { $line = trim($line); - return empty($line) || $this->isASqlComment($line); } protected function loadStream(?string $connectionName = null): void { - $directory = (new TemporaryDirectory(config('db-snapshots.temporary_directory_path')))->create(); + $temporaryDirectory = (new TemporaryDirectory(config('db-snapshots.temporary_directory_path')))->create(); + + $this->configureFilesystemDisk($temporaryDirectory->path()); + + $localDisk = $this->filesystemFactory->disk(self::class); + + try { + $this->processStream($localDisk, $connectionName); + } finally { + $temporaryDirectory->delete(); + } + } + private function configureFilesystemDisk(string $path): void + { config([ 'filesystems.disks.' . self::class => [ 'driver' => 'local', - 'root' => $directory->path(), + 'root' => $path, 'throw' => false, - ] + ], ]); + } - $localDisk = $this->filesystemFactory->disk(self::class); + private function processStream(FilesystemAdapter $localDisk, ?string $connectionName): void + { + $this->copyStreamToLocalDisk($localDisk); + + $stream = $this->openStream($localDisk); try { - LazyCollection::make(function () use ($localDisk) { - $localDisk->writeStream($this->fileName, $this->disk->readStream($this->fileName)); - - $stream = $this->compressionExtension === 'gz' - ? gzopen($localDisk->path($this->fileName), 'r') - : $localDisk->readStream($this->fileName); - - $statement = ''; - while (!feof($stream)) { - $chunk = $this->compressionExtension === 'gz' - ? gzread($stream, self::STREAM_BUFFER_SIZE) - : fread($stream, self::STREAM_BUFFER_SIZE); - - $lines = explode("\n", $chunk); - foreach ($lines as $idx => $line) { - if ($this->shouldIgnoreLine($line)) { - continue; - } - - $statement .= $line; - - // Carry-over the last line to the next chunk since it - // is possible that this chunk finished mid-line right on - // a semi-colon. - if (count($lines) == $idx + 1) { - break; - } - - if (str_ends_with(trim($statement), ';')) { - yield $statement; - $statement = ''; - } - } + $this->processStatements($stream, $connectionName); + } finally { + $this->closeStream($stream); + } + } + + private function copyStreamToLocalDisk(FilesystemAdapter $localDisk): void + { + $localDisk->writeStream($this->fileName, $this->disk->readStream($this->fileName)); + } + + private function openStream(FilesystemAdapter $localDisk): mixed + { + return $this->compressionExtension === 'gz' + ? gzopen($localDisk->path($this->fileName), 'r') + : $localDisk->readStream($this->fileName); + } + + private function closeStream(mixed $stream): void + { + $this->compressionExtension === 'gz' ? gzclose($stream) : fclose($stream); + } + + private function processStatements(mixed $stream, ?string $connectionName): void + { + $statement = ''; + while (!feof($stream)) { + $chunk = $this->readChunk($stream); + $lines = explode("\n", $chunk); + + foreach ($lines as $idx => $line) { + if ($this->shouldIgnoreLine($line)) { + continue; } - if (str_ends_with(trim($statement), ';')) { - yield $statement; + $statement .= $line; + + if ($this->isLastLineOfChunk($lines, $idx)) { + break; } - if ($this->compressionExtension === 'gz') { - gzclose($stream); - } else { - fclose($stream); + if ($this->isCompleteStatement($statement)) { + DB::connection($connectionName)->unprepared($statement); + $statement = ''; } - })->each(function (string $statement) use ($connectionName) { - DB::connection($connectionName)->unprepared($statement); - }); - } finally { - $directory->delete(); + } + } + + if ($this->isCompleteStatement($statement)) { + DB::connection($connectionName)->unprepared($statement); } } + private function readChunk(mixed $stream): string + { + return $this->compressionExtension === 'gz' + ? gzread($stream, self::STREAM_BUFFER_SIZE) + : fread($stream, self::STREAM_BUFFER_SIZE); + } + + private function isLastLineOfChunk(array $lines, int $idx): bool + { + return count($lines) === $idx + 1; + } + + private function isCompleteStatement(string $statement): bool + { + return str_ends_with(trim($statement), ';'); + } + public function delete(): void { event(new DeletingSnapshot($this)); - $this->disk->delete($this->fileName); - event(new DeletedSnapshot($this->fileName, $this->disk)); } diff --git a/tests/Commands/LoadTest.php b/tests/Commands/LoadTest.php index eb21cb6..c6ae51a 100644 --- a/tests/Commands/LoadTest.php +++ b/tests/Commands/LoadTest.php @@ -1,9 +1,14 @@ expectException(Exception::class); + + $disk = m::mock(FilesystemAdapter::class); + $disk->shouldReceive('exists') + ->with('nonexistent.sql') + ->andReturn(false); + + $snapshot = new Snapshot($disk, 'nonexistent.sql'); + $snapshot->load(); +}); + +it('throws an error for invalid SQL in snapshot', function () { + $disk = m::mock(FilesystemAdapter::class); + $disk->shouldReceive('get') + ->andReturn("INVALID SQL;\n"); + + $snapshot = new Snapshot($disk, 'invalid.sql'); + + $this->expectException(Exception::class); + $snapshot->load(); +}); + +it('deletes the snapshot and triggers event', function () { + Event::fake(); + + $disk = m::mock(FilesystemAdapter::class); + $disk->shouldReceive('delete') + ->once() + ->with('snapshot.sql') + ->andReturn(true); + + $snapshot = new Snapshot($disk, 'snapshot.sql'); + $snapshot->delete(); + + Event::assertDispatched(DeletedSnapshot::class, function ($event) use ($snapshot) { + return $event->fileName === $snapshot->fileName && $event->disk === $snapshot->disk; + }); +}); + +it('returns the correct size of the snapshot', function () { + $disk = m::mock(FilesystemAdapter::class); + $disk->shouldReceive('size') + ->andReturn(2048); + + $snapshot = new Snapshot($disk, 'snapshot.sql'); + + assertEquals(2048, $snapshot->size()); +}); + +it('returns the correct creation date of the snapshot', function () { + $timestamp = Carbon::now()->timestamp; + + $disk = m::mock(FilesystemAdapter::class); + $disk->shouldReceive('lastModified') + ->andReturn($timestamp); + + $snapshot = new Snapshot($disk, 'snapshot.sql'); + + assertEquals(Carbon::createFromTimestamp($timestamp), $snapshot->createdAt()); +}); + +it('handles empty snapshots gracefully', function () { + $disk = m::mock(FilesystemAdapter::class); + $disk->shouldReceive('get') + ->andReturn(""); + + $snapshot = new Snapshot($disk, 'empty.sql'); + + $snapshot->load(); + + // Expect no SQL to be executed + DB::shouldReceive('unprepared') + ->never(); +}); + +it('drops all current tables when requested', function () { + // Mock SchemaBuilder + $schemaBuilderMock = m::mock(); + $schemaBuilderMock->shouldReceive('dropAllTables')->once(); + + // Mock DB facade + DB::shouldReceive('connection') + ->andReturnSelf(); // Returns the DB connection + DB::shouldReceive('getSchemaBuilder') + ->andReturn($schemaBuilderMock); // Returns the mocked schema builder + DB::shouldReceive('getDefaultConnection') + ->andReturn('testing'); // Returns a mock default connection + DB::shouldReceive('reconnect')->once(); + + // Instance of Snapshot + $snapshot = new Snapshot(m::mock(FilesystemAdapter::class), 'snapshot.sql'); + + // Access protected method via Reflection + $reflection = new ReflectionMethod(Snapshot::class, 'dropAllCurrentTables'); + $reflection->setAccessible(true); + + // Invoke the protected method + $reflection->invoke($snapshot); +}); From 936561f04eb9c8b35a11fd227318cd5e13c684e2 Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Tue, 31 Dec 2024 10:11:34 +0100 Subject: [PATCH 4/6] Add return value ressource --- src/Snapshot.php | 8 ++++---- tests/Commands/LoadTest.php | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Snapshot.php b/src/Snapshot.php index 4f1c596..8741d59 100644 --- a/src/Snapshot.php +++ b/src/Snapshot.php @@ -137,19 +137,19 @@ private function copyStreamToLocalDisk(FilesystemAdapter $localDisk): void $localDisk->writeStream($this->fileName, $this->disk->readStream($this->fileName)); } - private function openStream(FilesystemAdapter $localDisk): mixed + private function openStream(FilesystemAdapter $localDisk): resource { return $this->compressionExtension === 'gz' ? gzopen($localDisk->path($this->fileName), 'r') : $localDisk->readStream($this->fileName); } - private function closeStream(mixed $stream): void + private function closeStream(resource $stream): void { $this->compressionExtension === 'gz' ? gzclose($stream) : fclose($stream); } - private function processStatements(mixed $stream, ?string $connectionName): void + private function processStatements(resource $stream, ?string $connectionName): void { $statement = ''; while (!feof($stream)) { @@ -179,7 +179,7 @@ private function processStatements(mixed $stream, ?string $connectionName): void } } - private function readChunk(mixed $stream): string + private function readChunk(resource $stream): string { return $this->compressionExtension === 'gz' ? gzread($stream, self::STREAM_BUFFER_SIZE) diff --git a/tests/Commands/LoadTest.php b/tests/Commands/LoadTest.php index c6ae51a..d2e87ba 100644 --- a/tests/Commands/LoadTest.php +++ b/tests/Commands/LoadTest.php @@ -249,3 +249,4 @@ function getNameOfLoadedSnapshot(): string // Invoke the protected method $reflection->invoke($snapshot); }); + From 50c0b5c8ce4ccc9dc42a224071e096225f993e8c Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Tue, 31 Dec 2024 10:27:36 +0100 Subject: [PATCH 5/6] Use mixed --- src/Snapshot.php | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/Snapshot.php b/src/Snapshot.php index 8741d59..9e300b9 100644 --- a/src/Snapshot.php +++ b/src/Snapshot.php @@ -137,19 +137,29 @@ private function copyStreamToLocalDisk(FilesystemAdapter $localDisk): void $localDisk->writeStream($this->fileName, $this->disk->readStream($this->fileName)); } - private function openStream(FilesystemAdapter $localDisk): resource + private function openStream(FilesystemAdapter $localDisk): mixed { - return $this->compressionExtension === 'gz' + $stream = $this->compressionExtension === 'gz' ? gzopen($localDisk->path($this->fileName), 'r') : $localDisk->readStream($this->fileName); + + if (!is_resource($stream)) { + throw new \RuntimeException("Failed to open stream for file: {$this->fileName}"); + } + + return $stream; } - private function closeStream(resource $stream): void + private function closeStream(mixed $stream): void { + if (!is_resource($stream)) { + throw new \RuntimeException("Invalid stream provided for closing."); + } + $this->compressionExtension === 'gz' ? gzclose($stream) : fclose($stream); } - private function processStatements(resource $stream, ?string $connectionName): void + private function processStatements(mixed $stream, ?string $connectionName): void { $statement = ''; while (!feof($stream)) { @@ -179,7 +189,7 @@ private function processStatements(resource $stream, ?string $connectionName): v } } - private function readChunk(resource $stream): string + private function readChunk(mixed $stream): string { return $this->compressionExtension === 'gz' ? gzread($stream, self::STREAM_BUFFER_SIZE) From 4e2ca33efaa4eaf196820f4c179b35d3b0ed175a Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Thu, 13 Feb 2025 10:18:22 +0100 Subject: [PATCH 6/6] Update Snapshot.php --- src/Snapshot.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Snapshot.php b/src/Snapshot.php index 9e300b9..62f86cc 100644 --- a/src/Snapshot.php +++ b/src/Snapshot.php @@ -192,7 +192,7 @@ private function processStatements(mixed $stream, ?string $connectionName): void private function readChunk(mixed $stream): string { return $this->compressionExtension === 'gz' - ? gzread($stream, self::STREAM_BUFFER_SIZE) + ? gzgets($stream, self::STREAM_BUFFER_SIZE) : fread($stream, self::STREAM_BUFFER_SIZE); }