Merge pull request #2 from Naoray/feature/added-deduplication-stores

Feature/added deduplication stores

Author: Naoray

.github/workflows/run-tests.yml

@@ -13,6 +13,18 @@ jobs:
   test:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 5
+
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     strategy:
       fail-fast: true
       matrix:
@@ -34,6 +46,7 @@ jobs:
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ matrix.php }}
+          extensions: redis
           coverage: none
 
       - name: Setup problem matchers
@@ -51,3 +64,6 @@ jobs:
 
       - name: Execute tests
         run: vendor/bin/pest --ci
+        env:
+          REDIS_HOST: localhost
+          REDIS_PORT: 6379

README.md

@@ -5,7 +5,7 @@
 [![GitHub Code Style Action Status](https://img.shields.io/github/actions/workflow/status/naoray/laravel-github-monolog/fix-php-code-style-issues.yml?branch=main&label=code%20style&style=flat-square)](https://github.com/naoray/laravel-github-monolog/actions?query=workflow%3A"Fix+PHP+code+style+issues"+branch%3Amain)
 [![Total Downloads](https://img.shields.io/packagist/dt/naoray/laravel-github-monolog.svg?style=flat-square)](https://packagist.org/packages/naoray/laravel-github-monolog)
 
-A Laravel package that automatically creates GitHub issues from your application logs. Perfect for smaller projects where full-featured logging services like Sentry or Bugsnag might be overkill, but you still want to track bugs effectively.
+Automatically create GitHub issues from your Laravel logs. Perfect for smaller projects without the need for full-featured logging services.
 
 ## Requirements
 
@@ -15,10 +15,10 @@ A Laravel package that automatically creates GitHub issues from your application
 
 ## Features
 
-- ✨ Automatically creates GitHub issues from log entries
-- 🔍 Intelligently groups similar errors into single issues
-- 💬 Adds comments to existing issues for recurring errors
-- 🏷️ Supports customizable labels for efficient organization
+- ✨ Automatically create GitHub issues from logs
+- 🔍 Group similar errors into single issues
+- 💬 Add comments to existing issues for recurring errors
+- 🏷️ Support customizable labels
 - 🎯 Smart deduplication to prevent issue spam
 - ⚡️ Buffered logging for better performance
 
@@ -38,15 +38,15 @@ If the same error occurs again, instead of creating a duplicate, a new comment i
 
 ## Installation
 
-Install the package via composer:
+Install with Composer:
 
 ```bash
 composer require naoray/laravel-github-monolog
 ```
 
 ## Configuration
 
-Add the GitHub logging channel to your `config/logging.php`:
+Add the GitHub logging channel to `config/logging.php`:
 
 ```php
 'channels' => [
@@ -62,10 +62,6 @@ Add the GitHub logging channel to your `config/logging.php`:
         // Optional configuration
         'level' => env('LOG_LEVEL', 'error'),
         'labels' => ['bug'],
-        'deduplication' => [
-            'store' => storage_path('logs/github-issues-dedup.log'),  // Custom path for deduplication storage
-            'time' => 60,  // Time in seconds to consider logs as duplicates
-        ],
     ],
 ]
 ```
@@ -79,6 +75,102 @@ GITHUB_TOKEN=your-github-personal-access-token
 
 You can use the `github` log channel as your default `LOG_CHANNEL` or add it as part of your stack in `LOG_STACK`.
 
+### Advanced Configuration
+
+Deduplication and buffering are enabled by default to enhance logging. Customize these features to suit your needs.
+
+#### Deduplication
+
+Group similar errors to avoid duplicate issues. By default, the package uses file-based storage. Customize the storage and time window to fit your application.
+
+```php
+'github' => [
+    // ... basic config from above ...
+    'deduplication' => [
+        'store' => 'file',  // Default store
+        'time' => 60,       // Time window in seconds
+    ],
+]
+```
+
+##### Alternative Storage Options
+
+Consider other storage options in these Laravel-specific scenarios:
+
+- **Redis Store**: Use when:
+  - Running async queue jobs (file storage won't work across processes)
+  - Using Laravel Horizon for queue management
+  - Running multiple application instances behind a load balancer
+
+  ```php
+  'deduplication' => [
+      'store' => 'redis',
+      'prefix' => 'github-monolog:',
+      'connection' => 'default',  // Uses your Laravel Redis connection
+  ],
+  ```
+
+- **Database Store**: Use when:
+  - Running queue jobs but Redis isn't available
+  - Need to persist deduplication data across deployments
+  - Want to query/debug deduplication history via database
+
+  ```php
+  'deduplication' => [
+      'store' => 'database',
+      'table' => 'github_monolog_deduplication',
+      'connection' => null,  // Uses your default database connection
+  ],
+  ```
+
+#### Buffering
+
+Buffer logs to reduce GitHub API calls. Customize the buffer size and overflow behavior to optimize performance:
+
+```php
+'github' => [
+    // ... basic config from above ...
+    'buffer' => [
+        'limit' => 0,        // Maximum records in buffer (0 = unlimited, flush on shutdown)
+        'flush_on_overflow' => true,  // When limit is reached: true = flush all, false = remove oldest
+    ],
+]
+```
+
+When buffering is active:
+- Logs are collected in memory until flushed
+- Buffer is automatically flushed on application shutdown
+- When limit is reached:
+  - With `flush_on_overflow = true`: All records are flushed
+  - With `flush_on_overflow = false`: Only the oldest record is removed
+
+#### Signature Generator
+
+Control how errors are grouped by customizing the signature generator. By default, the package uses a generator that creates signatures based on exception details or log message content.
+
+```php
+'github' => [
+    // ... basic config from above ...
+    'signature_generator' => \Naoray\LaravelGithubMonolog\Deduplication\DefaultSignatureGenerator::class,
+]
+```
+
+You can implement your own signature generator by implementing the `SignatureGeneratorInterface`:
+
+```php
+use Monolog\LogRecord;
+use Naoray\LaravelGithubMonolog\Deduplication\SignatureGeneratorInterface;
+
+class CustomSignatureGenerator implements SignatureGeneratorInterface
+{
+    public function generate(LogRecord $record): string
+    {
+        // Your custom logic to generate a signature
+        return md5($record->message);
+    }
+}
+```
+
 ### Getting a GitHub Token
 
 To obtain a Personal Access Token:
@@ -105,22 +197,6 @@ Log::channel('github')->error('Something went wrong!');
 Log::stack(['daily', 'github'])->error('Something went wrong!');
 ```
 
-### Deduplication
-
-The package includes smart deduplication to prevent your repository from being flooded with duplicate issues:
-
-1. **Time-based Deduplication**: Similar errors within the configured time window (default: 60 seconds) are considered duplicates
-2. **Intelligent Grouping**: Uses error signatures to group similar errors, even if the exact details differ
-3. **Automatic Storage**: Deduplication data is automatically stored in your Laravel logs directory
-4. **Configurable**: Customize both the storage location and deduplication time window
-
-For example, if your application encounters the same error multiple times in quick succession:
-- First occurrence: Creates a new GitHub issue
-- Subsequent occurrences within the deduplication window: No new issues created
-- After the deduplication window: Creates a new issue or adds a comment to the existing one
-
-This helps keep your GitHub issues organized and prevents notification spam during error storms.
-
 ## Testing
 
 ```bash

composer.json

@@ -4,7 +4,9 @@
   "keywords": [
     "Krishan Koenig",
     "laravel",
-    "laravel-github-monolog"
+    "monolog",
+    "github",
+    "logging"
   ],
   "homepage": "https://github.com/naoray/laravel-github-monolog",
   "license": "MIT",
@@ -37,13 +39,12 @@
   },
   "autoload": {
     "psr-4": {
-      "Naoray\\LaravelGithubMonolog\\": "src/"
+      "Naoray\\LaravelGithubMonolog\\": "src"
     }
   },
   "autoload-dev": {
     "psr-4": {
-      "Naoray\\GithubMonolog\\Tests\\": "tests/",
-      "Workbench\\App\\": "workbench/app/"
+      "Naoray\\LaravelGithubMonolog\\Tests\\": "tests"
     }
   },
   "scripts": {

src/Deduplication/DeduplicationHandler.php

@@ -0,0 +1,59 @@
+<?php
+
+namespace Naoray\LaravelGithubMonolog\Deduplication;
+
+use Illuminate\Support\Collection;
+use Monolog\Handler\BufferHandler;
+use Monolog\Handler\HandlerInterface;
+use Monolog\Level;
+use Monolog\LogRecord;
+use Naoray\LaravelGithubMonolog\Deduplication\Stores\StoreInterface;
+
+class DeduplicationHandler extends BufferHandler
+{
+    public function __construct(
+        HandlerInterface $handler,
+        protected StoreInterface $store,
+        protected SignatureGeneratorInterface $signatureGenerator,
+        int|string|Level $level = Level::Error,
+        int $bufferLimit = 0,
+        bool $bubble = true,
+        bool $flushOnOverflow = false,
+    ) {
+        parent::__construct(
+            handler: $handler,
+            bufferLimit: $bufferLimit,
+            level: $level,
+            bubble: $bubble,
+            flushOnOverflow: $flushOnOverflow,
+        );
+    }
+
+    public function flush(): void
+    {
+        if ($this->bufferSize === 0) {
+            return;
+        }
+
+        collect($this->buffer)
+            ->map(function (LogRecord $record) {
+                $signature = $this->signatureGenerator->generate($record);
+
+                // Create new record with signature in extra data
+                $record = $record->with(extra: ['github_issue_signature' => $signature] + $record->extra);
+
+                // If the record is a duplicate, we don't want to add it to the store
+                if ($this->store->isDuplicate($record, $signature)) {
+                    return null;
+                }
+
+                $this->store->add($record, $signature);
+
+                return $record;
+            })
+            ->filter()
+            ->pipe(fn (Collection $records) => $this->handler->handleBatch($records->toArray()));
+
+        $this->clear();
+    }
+}

src/DefaultSignatureGenerator.php renamed to src/Deduplication/DefaultSignatureGenerator.php

@@ -1,12 +1,11 @@
 <?php
 
-namespace Naoray\LaravelGithubMonolog;
+namespace Naoray\LaravelGithubMonolog\Deduplication;
 
 use Monolog\LogRecord;
-use Naoray\LaravelGithubMonolog\Contracts\SignatureGenerator;
 use Throwable;
 
-class DefaultSignatureGenerator implements SignatureGenerator
+class DefaultSignatureGenerator implements SignatureGeneratorInterface
 {
     /**
      * Generate a unique signature for the log record

src/Contracts/SignatureGenerator.php renamed to src/Deduplication/SignatureGeneratorInterface.php

@@ -1,10 +1,10 @@
 <?php
 
-namespace Naoray\LaravelGithubMonolog\Contracts;
+namespace Naoray\LaravelGithubMonolog\Deduplication;
 
 use Monolog\LogRecord;
 
-interface SignatureGenerator
+interface SignatureGeneratorInterface
 {
     /**
      * Generate a unique signature for the log record

src/Deduplication/Stores/AbstractStore.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Naoray\LaravelGithubMonolog\Deduplication\Stores;
+
+use Illuminate\Support\Carbon;
+use Monolog\LogRecord;
+
+abstract class AbstractStore implements StoreInterface
+{
+    public function __construct(
+        protected int $time = 60
+    ) {}
+
+    protected function buildEntry(string $signature, int $timestamp): string
+    {
+        return $timestamp.':'.$signature;
+    }
+
+    public function isDuplicate(LogRecord $record, string $signature): bool
+    {
+        $foundDuplicate = false;
+
+        foreach ($this->get() as $entry) {
+            [$timestamp, $storedSignature] = explode(':', $entry, 2);
+            $timestamp = (int) $timestamp;
+
+            if ($this->isExpired($timestamp)) {
+                continue;
+            }
+
+            if ($storedSignature === $signature) {
+                $foundDuplicate = true;
+            }
+        }
+
+        return $foundDuplicate;
+    }
+
+    protected function isExpired(int $timestamp): bool
+    {
+        return $this->getTimestampValidity() > $timestamp;
+    }
+
+    protected function getTimestampValidity(): int
+    {
+        return $this->getTimestamp() - $this->time;
+    }
+
+    protected function getTimestamp(): int
+    {
+        return Carbon::now()->timestamp;
+    }
+}