From 18f6b188a885bee5be2548f61f5ca51b65fc7284 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 6 Apr 2025 02:56:18 -0500 Subject: [PATCH 1/6] Add file watcher and checksum functionality - Add FileWatcher to monitor files and directories for changes - Add WatchRecursive for recursive directory monitoring - Implement Checksum function for file hash calculation with multiple algorithms - Add enum package for EventType and HashAlg enumerations - Update CI workflow to use latest golangci-lint - Add comprehensive documentation and examples - Fix code to comply with linter requirements --- .github/workflows/ci.yml | 4 +- .golangci.yml | 124 ++++++++++---------- README.md | 82 +++++++++++++ enum/event_type_enum.go | 119 +++++++++++++++++++ enum/hash_alg_enum.go | 134 ++++++++++++++++++++++ file_watcher.go | 207 +++++++++++++++++++++++++++++++++ file_watcher_test.go | 241 +++++++++++++++++++++++++++++++++++++++ fileutils.go | 93 +++++++++++++-- fileutils_test.go | 96 +++++++++++++++- go.mod | 6 +- go.sum | 4 + 11 files changed, 1037 insertions(+), 73 deletions(-) create mode 100644 enum/event_type_enum.go create mode 100644 enum/hash_alg_enum.go create mode 100644 file_watcher.go create mode 100644 file_watcher_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f758d69..5154f2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,9 +29,9 @@ jobs: cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v7 with: - version: latest + version: v2.0.2 - name: install goveralls run: | diff --git a/.golangci.yml b/.golangci.yml index 81bbe49..30db72d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,65 +1,71 @@ -linters-settings: - govet: - enable: - - shadow - golint: - min-confidence: 0.8 - gocyclo: - min-complexity: 15 - maligned: - suggest-new: true - dupl: - threshold: 100 - goconst: - min-len: 2 - min-occurrences: 2 - misspell: - locale: US - lll: - line-length: 140 - gocritic: - enabled-tags: - - performance - - style - - experimental - disabled-checks: - - wrapperFunc - - hugeParam - - rangeValCopy - +version: "2" +run: + tests: false linters: - disable-all: true + default: none enable: - - megacheck - - revive + - copyloopvar + - dupl + - gochecknoinits + - gocritic + - gosec - govet - - unconvert - - megacheck - - unused - - gas - - gocyclo - - misspell - - unparam - - typecheck - ineffassign - - stylecheck - - gochecknoinits + - misspell - nakedret - - gosimple - prealloc - - fast: false - - -run: - # modules-download-mode: vendor - skip-dirs: - - vendor - concurrency: 4 - -issues: - exclude-rules: - - text: "weak cryptographic primitive" - linters: - - gosec - exclude-use-default: false + - revive + - staticcheck + - unconvert + - unparam + - unused + settings: + goconst: + min-len: 2 + min-occurrences: 2 + gocritic: + disabled-checks: + - wrapperFunc + - hugeParam + - rangeValCopy + - singleCaseSwitch + - ifElseChain + enabled-tags: + - performance + - style + - experimental + govet: + enable: + - shadow + lll: + line-length: 140 + misspell: + locale: US + exclusions: + generated: lax + rules: + - linters: + - staticcheck + text: at least one file in a package should have a package comment + - linters: + - revive + text: should have a package comment + - linters: + - dupl + - gosec + path: _test\.go + paths: + - vendor + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + exclusions: + generated: lax + paths: + - vendor + - third_party$ + - builtin$ + - examples$ diff --git a/README.md b/README.md index a7b16ff..c723116 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,88 @@ Package `fileutils` provides useful, high-level file operations. - `TempFileName` returns a new temporary file name using secure random generation - `SanitizePath` cleans file path - `TouchFile` creates an empty file or updates timestamps of existing one +- `Checksum` calculates file checksum using various hash algorithms (MD5, SHA1, SHA256, etc.) +- `FileWatcher` watches files or directories for changes +- `WatchRecursive` watches a directory recursively for changes + +## Usage Examples + +### File Operations + +```go +// Copy a file +err := fileutils.CopyFile("source.txt", "destination.txt") +if err != nil { + log.Fatalf("Failed to copy file: %v", err) +} + +// Move a file +err = fileutils.MoveFile("source.txt", "destination.txt") +if err != nil { + log.Fatalf("Failed to move file: %v", err) +} + +// Check if a file or directory exists +if fileutils.IsFile("file.txt") { + fmt.Println("File exists") +} +if fileutils.IsDir("directory") { + fmt.Println("Directory exists") +} + +// Generate a temporary file name +tempName, err := fileutils.TempFileName("/tmp", "prefix-*.ext") +if err != nil { + log.Fatalf("Failed to generate temp file name: %v", err) +} +fmt.Println("Temp file:", tempName) +``` + +### File Checksum + +```go +// Calculate MD5 checksum +md5sum, err := fileutils.Checksum("path/to/file", enum.HashAlgMD5) +if err != nil { + log.Fatalf("Failed to calculate MD5: %v", err) +} +fmt.Printf("MD5: %s\n", md5sum) + +// Calculate SHA256 checksum +sha256sum, err := fileutils.Checksum("path/to/file", enum.HashAlgSHA256) +if err != nil { + log.Fatalf("Failed to calculate SHA256: %v", err) +} +fmt.Printf("SHA256: %s\n", sha256sum) +``` + +### File Watcher + +```go +// Create a simple file watcher +watcher, err := fileutils.NewFileWatcher("/path/to/file", func(event FileEvent) { + fmt.Printf("Event: %s, Path: %s\n", event.Type, event.Path) +}) +if err != nil { + log.Fatalf("Failed to create watcher: %v", err) +} +defer watcher.Close() + +// Watch a directory recursively +watcher, err := fileutils.WatchRecursive("/path/to/dir", func(event FileEvent) { + fmt.Printf("Event: %s, Path: %s\n", event.Type, event.Path) +}) +if err != nil { + log.Fatalf("Failed to create watcher: %v", err) +} +defer watcher.Close() + +// Add another path to an existing watcher +err = watcher.AddPath("/path/to/another/file") + +// Remove a path from the watcher +err = watcher.RemovePath("/path/to/file") +``` ## Install and update diff --git a/enum/event_type_enum.go b/enum/event_type_enum.go new file mode 100644 index 0000000..ba7c194 --- /dev/null +++ b/enum/event_type_enum.go @@ -0,0 +1,119 @@ +// Code generated by enum generator; DO NOT EDIT. +package enum + +import ( + "fmt" + + "database/sql/driver" + "strings" +) + +// EventType is the exported type for the enum +type EventType struct { + name string + value int +} + +func (e EventType) String() string { return e.name } + +// MarshalText implements encoding.TextMarshaler +func (e EventType) MarshalText() ([]byte, error) { + return []byte(e.name), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler +func (e *EventType) UnmarshalText(text []byte) error { + var err error + *e, err = ParseEventType(string(text)) + return err +} + +// Value implements the driver.Valuer interface +func (e EventType) Value() (driver.Value, error) { + return e.name, nil +} + +// Scan implements the sql.Scanner interface +func (e *EventType) Scan(value interface{}) error { + if value == nil { + *e = EventTypeValues()[0] + return nil + } + + str, ok := value.(string) + if !ok { + if b, ok := value.([]byte); ok { + str = string(b) + } else { + return fmt.Errorf("invalid eventType value: %v", value) + } + } + + val, err := ParseEventType(str) + if err != nil { + return err + } + + *e = val + return nil +} + +// ParseEventType converts string to eventType enum value +func ParseEventType(v string) (EventType, error) { + + switch strings.ToLower(v) { + case strings.ToLower("Chmod"): + return EventTypeChmod, nil + case strings.ToLower("Create"): + return EventTypeCreate, nil + case strings.ToLower("Remove"): + return EventTypeRemove, nil + case strings.ToLower("Rename"): + return EventTypeRename, nil + case strings.ToLower("Write"): + return EventTypeWrite, nil + + } + + return EventType{}, fmt.Errorf("invalid eventType: %s", v) +} + +// MustEventType is like ParseEventType but panics if string is invalid +func MustEventType(v string) EventType { + r, err := ParseEventType(v) + if err != nil { + panic(err) + } + return r +} + +// Public constants for eventType values +var ( + EventTypeChmod = EventType{name: "Chmod", value: 4} + EventTypeCreate = EventType{name: "Create", value: 0} + EventTypeRemove = EventType{name: "Remove", value: 2} + EventTypeRename = EventType{name: "Rename", value: 3} + EventTypeWrite = EventType{name: "Write", value: 1} +) + +// EventTypeValues returns all possible enum values +func EventTypeValues() []EventType { + return []EventType{ + EventTypeChmod, + EventTypeCreate, + EventTypeRemove, + EventTypeRename, + EventTypeWrite, + } +} + +// EventTypeNames returns all possible enum names +func EventTypeNames() []string { + return []string{ + "Chmod", + "Create", + "Remove", + "Rename", + "Write", + } +} diff --git a/enum/hash_alg_enum.go b/enum/hash_alg_enum.go new file mode 100644 index 0000000..b1fa75f --- /dev/null +++ b/enum/hash_alg_enum.go @@ -0,0 +1,134 @@ + // Code generated by enum generator; DO NOT EDIT. +package enum + +import ( + "fmt" + + "database/sql/driver" + "strings" +) + +// HashAlg is the exported type for the enum +type HashAlg struct { + name string + value int +} + +func (e HashAlg) String() string { return e.name } + +// MarshalText implements encoding.TextMarshaler +func (e HashAlg) MarshalText() ([]byte, error) { + return []byte(e.name), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler +func (e *HashAlg) UnmarshalText(text []byte) error { + var err error + *e, err = ParseHashAlg(string(text)) + return err +} + +// Value implements the driver.Valuer interface +func (e HashAlg) Value() (driver.Value, error) { + return e.name, nil +} + +// Scan implements the sql.Scanner interface +func (e *HashAlg) Scan(value interface{}) error { + if value == nil { + *e = HashAlgValues()[0] + return nil + } + + str, ok := value.(string) + if !ok { + if b, ok := value.([]byte); ok { + str = string(b) + } else { + return fmt.Errorf("invalid hashAlg value: %v", value) + } + } + + val, err := ParseHashAlg(str) + if err != nil { + return err + } + + *e = val + return nil +} + +// ParseHashAlg converts string to hashAlg enum value +func ParseHashAlg(v string) (HashAlg, error) { + + switch strings.ToLower(v) { + case strings.ToLower("MD5"): + return HashAlgMD5, nil + case strings.ToLower("SHA1"): + return HashAlgSHA1, nil + case strings.ToLower("SHA224"): + return HashAlgSHA224, nil + case strings.ToLower("SHA256"): + return HashAlgSHA256, nil + case strings.ToLower("SHA384"): + return HashAlgSHA384, nil + case strings.ToLower("SHA512"): + return HashAlgSHA512, nil + case strings.ToLower("SHA512_224"): + return HashAlgSHA512_224, nil + case strings.ToLower("SHA512_256"): + return HashAlgSHA512_256, nil + + } + + return HashAlg{}, fmt.Errorf("invalid hashAlg: %s", v) +} + +// MustHashAlg is like ParseHashAlg but panics if string is invalid +func MustHashAlg(v string) HashAlg { + r, err := ParseHashAlg(v) + if err != nil { + panic(err) + } + return r +} + +// Public constants for hashAlg values +var ( + HashAlgMD5 = HashAlg{name: "MD5", value: 0} + HashAlgSHA1 = HashAlg{name: "SHA1", value: 1} + HashAlgSHA224 = HashAlg{name: "SHA224", value: 3} + HashAlgSHA256 = HashAlg{name: "SHA256", value: 2} + HashAlgSHA384 = HashAlg{name: "SHA384", value: 4} + HashAlgSHA512 = HashAlg{name: "SHA512", value: 5} + HashAlgSHA512_224 = HashAlg{name: "SHA512_224", value: 6} + HashAlgSHA512_256 = HashAlg{name: "SHA512_256", value: 7} +) + +// HashAlgValues returns all possible enum values +func HashAlgValues() []HashAlg { + return []HashAlg{ + HashAlgMD5, + HashAlgSHA1, + HashAlgSHA224, + HashAlgSHA256, + HashAlgSHA384, + HashAlgSHA512, + HashAlgSHA512_224, + HashAlgSHA512_256, + } +} + +// HashAlgNames returns all possible enum names +func HashAlgNames() []string { + return []string{ + "MD5", + "SHA1", + "SHA224", + "SHA256", + "SHA384", + "SHA512", + "SHA512_224", + "SHA512_256", + } +} diff --git a/file_watcher.go b/file_watcher.go new file mode 100644 index 0000000..038072b --- /dev/null +++ b/file_watcher.go @@ -0,0 +1,207 @@ +package fileutils + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/fsnotify/fsnotify" + + "github.com/go-pkgz/fileutils/enum" +) + +//go:generate enum -type=eventType -path=enum + +// eventType represents the type of file system event +// +//nolint:unused // This type is used by the enum generator +type eventType int + +// Event types +// +//nolint:unused // These constants are used by the enum generator +const ( + eventTypeCreate eventType = iota + 1 + eventTypeWrite + eventTypeRemove + eventTypeRename + eventTypeChmod +) + +// FileEvent represents a file system event +type FileEvent struct { + Path string // path to the file or directory + Type enum.EventType // type of event +} + +// FileWatcher watches for file system events +type FileWatcher struct { + watcher *fsnotify.Watcher + callback func(FileEvent) + done chan struct{} +} + +// NewFileWatcher creates a new file watcher for the specified path +func NewFileWatcher(path string, callback func(FileEvent)) (*FileWatcher, error) { + if path == "" { + return nil, errors.New("empty path") + } + + if callback == nil { + return nil, errors.New("callback function is required") + } + + // check if path exists + if !IsFile(path) && !IsDir(path) { + return nil, fmt.Errorf("path does not exist: %s", path) + } + + // create fsnotify watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, fmt.Errorf("failed to create watcher: %w", err) + } + + // add path to watcher + if err := watcher.Add(path); err != nil { + _ = watcher.Close() + return nil, fmt.Errorf("failed to watch path %s: %w", path, err) + } + + fw := &FileWatcher{ + watcher: watcher, + callback: callback, + done: make(chan struct{}), + } + + // start watching in a goroutine + go fw.watch() + + return fw, nil +} + +// watch processes events from the fsnotify watcher +func (fw *FileWatcher) watch() { + for { + select { + case event, ok := <-fw.watcher.Events: + if !ok { + return + } + + // convert fsnotify event to our event type + var eventType enum.EventType + if event.Has(fsnotify.Create) { + eventType = enum.EventTypeCreate + } else if event.Has(fsnotify.Write) { + eventType = enum.EventTypeWrite + } else if event.Has(fsnotify.Remove) { + eventType = enum.EventTypeRemove + } else if event.Has(fsnotify.Rename) { + eventType = enum.EventTypeRename + } else if event.Has(fsnotify.Chmod) { + eventType = enum.EventTypeChmod + } else { + continue // unknown event type + } + + // call the callback with the event + fw.callback(FileEvent{ + Path: event.Name, + Type: eventType, + }) + + case err, ok := <-fw.watcher.Errors: + if !ok { + return + } + // log error or call error callback + fmt.Printf("error: %v\n", err) + + case <-fw.done: + return + } + } +} + +// Close stops watching and releases resources +func (fw *FileWatcher) Close() error { + close(fw.done) + return fw.watcher.Close() +} + +// AddPath adds a path to the watcher +func (fw *FileWatcher) AddPath(path string) error { + if path == "" { + return errors.New("empty path") + } + + // check if path exists + if !IsFile(path) && !IsDir(path) { + return fmt.Errorf("path does not exist: %s", path) + } + + return fw.watcher.Add(path) +} + +// RemovePath removes a path from the watcher +func (fw *FileWatcher) RemovePath(path string) error { + if path == "" { + return errors.New("empty path") + } + + return fw.watcher.Remove(path) +} + +// WatchRecursive watches a directory recursively +func WatchRecursive(dir string, callback func(FileEvent)) (*FileWatcher, error) { + if dir == "" { + return nil, errors.New("empty directory path") + } + + if callback == nil { + return nil, errors.New("callback function is required") + } + + // check if directory exists + if !IsDir(dir) { + return nil, fmt.Errorf("directory does not exist: %s", dir) + } + + // create fsnotify watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, fmt.Errorf("failed to create watcher: %w", err) + } + + // create file watcher + fw := &FileWatcher{ + watcher: watcher, + callback: callback, + done: make(chan struct{}), + } + + // add all subdirectories + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + if err := watcher.Add(path); err != nil { + return fmt.Errorf("failed to watch directory %s: %w", path, err) + } + } + return nil + }) + + if err != nil { + _ = watcher.Close() + return nil, fmt.Errorf("failed to set up recursive watching: %w", err) + } + + // start watching in a goroutine + go fw.watch() + + return fw, nil +} diff --git a/file_watcher_test.go b/file_watcher_test.go new file mode 100644 index 0000000..2c348d1 --- /dev/null +++ b/file_watcher_test.go @@ -0,0 +1,241 @@ +package fileutils + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFileWatcher(t *testing.T) { + // skip this test on macOS as it's flaky due to how FSEvents works + if os.Getenv("GOOS") == "darwin" { + t.Skip("Skipping test on macOS") + } + + // create a temporary directory for testing + tmpDir := t.TempDir() + testFile := filepath.Join(tmpDir, "test.txt") + + // create a test file + err := os.WriteFile(testFile, []byte("initial content"), 0644) + require.NoError(t, err) + + // create a channel to receive events + eventCh := make(chan FileEvent, 10) // buffered channel to avoid blocking + + // create a file watcher + watcher, err := NewFileWatcher(testFile, func(event FileEvent) { + // send the event to the channel + select { + case eventCh <- event: + // event sent + default: + // channel full or closed, ignore + } + }) + require.NoError(t, err) + defer func() { _ = watcher.Close() }() + + // sleep a bit to let the watcher initialize + time.Sleep(100 * time.Millisecond) + + // modify the file to trigger an event + err = os.WriteFile(testFile, []byte("modified content"), 0644) + require.NoError(t, err) + + // wait for the event to be received or timeout + select { + case event := <-eventCh: + assert.Equal(t, testFile, event.Path) + // don't assert the exact event type as it can vary by OS + // just verify we got an event + case <-time.After(5 * time.Second): // longer timeout for slower systems + t.Fatal("Timeout waiting for file event") + } +} + +func TestWatchRecursive(t *testing.T) { + // skip this test on macOS as it's flaky due to how FSEvents works + if os.Getenv("GOOS") == "darwin" { + t.Skip("Skipping test on macOS") + } + + // create a temporary directory for testing + tmpDir := t.TempDir() + subDir := filepath.Join(tmpDir, "subdir") + err := os.Mkdir(subDir, 0755) + require.NoError(t, err) + + // create a buffered channel to receive events + eventCh := make(chan FileEvent, 10) // buffered channel to avoid blocking + + // create a recursive file watcher + watcher, err := WatchRecursive(tmpDir, func(event FileEvent) { + // send the event to the channel + select { + case eventCh <- event: + // event sent + default: + // channel full or closed, ignore + } + }) + require.NoError(t, err) + defer func() { _ = watcher.Close() }() + + // wait a bit to allow the watcher to initialize + time.Sleep(200 * time.Millisecond) + + // create a warmup file to ensure the watcher is ready + warmupFile := filepath.Join(tmpDir, "warmup.txt") + err = os.WriteFile(warmupFile, []byte("warmup content"), 0644) + require.NoError(t, err) + + // wait for any events from the warmup + time.Sleep(300 * time.Millisecond) + + // clear channel from warmup events + select { + case <-eventCh: + // discard warmup event + default: + // no event yet, continue + } + + // create a file in the subdirectory to trigger an event + testFile := filepath.Join(subDir, "test.txt") + err = os.WriteFile(testFile, []byte("test content"), 0644) + require.NoError(t, err) + + // wait for the event to be received or timeout + + select { + case event := <-eventCh: + assert.Equal(t, testFile, event.Path) + // don't assert the exact event type as it can vary by OS + // just verify we got an event + case <-time.After(5 * time.Second): + t.Fatal("Timeout waiting for file event") + } +} + +func TestFileWatcherAddPath(t *testing.T) { + // create a temporary directory for testing + tmpDir := t.TempDir() + testFile1 := filepath.Join(tmpDir, "test1.txt") + testFile2 := filepath.Join(tmpDir, "test2.txt") + + // create test files + err := os.WriteFile(testFile1, []byte("file 1"), 0644) + require.NoError(t, err) + err = os.WriteFile(testFile2, []byte("file 2"), 0644) + require.NoError(t, err) + + // create a channel to receive events + eventCh := make(chan FileEvent, 10) // buffered channel to avoid blocking + + // create a file watcher for the first file + watcher, err := NewFileWatcher(testFile1, func(event FileEvent) { + // send the event to the channel + select { + case eventCh <- event: + // event sent + default: + // channel full or closed, ignore + } + }) + require.NoError(t, err) + defer func() { _ = watcher.Close() }() + + // modify the first file to trigger an event + err = os.WriteFile(testFile1, []byte("modified file 1"), 0644) + require.NoError(t, err) + + // wait for the event to be received or timeout + select { + case event := <-eventCh: + assert.Equal(t, testFile1, event.Path) + // don't assert the exact event type as it can vary by OS + case <-time.After(5 * time.Second): + t.Fatal("Timeout waiting for file event") + } + + // add the second file to the watcher + err = watcher.AddPath(testFile2) + require.NoError(t, err) + + // modify the second file to trigger an event + err = os.WriteFile(testFile2, []byte("modified file 2"), 0644) + require.NoError(t, err) + + // wait for the event to be received or timeout + select { + case event := <-eventCh: + // just verify we got an event for one of our files + assert.True(t, event.Path == testFile1 || event.Path == testFile2, + "Expected event for either %s or %s, got %s", testFile1, testFile2, event.Path) + case <-time.After(5 * time.Second): + t.Fatal("Timeout waiting for file event") + } +} + +func TestFileWatcherRemovePath(t *testing.T) { + // skip this test on macOS as it's flaky due to how FSEvents works + if os.Getenv("GOOS") == "darwin" { + t.Skip("Skipping test on macOS") + } + + // create a temporary directory for testing + tmpDir := t.TempDir() + testFile1 := filepath.Join(tmpDir, "test1.txt") + testFile2 := filepath.Join(tmpDir, "test2.txt") + + // create test files + err := os.WriteFile(testFile1, []byte("file 1"), 0644) + require.NoError(t, err) + err = os.WriteFile(testFile2, []byte("file 2"), 0644) + require.NoError(t, err) + + // create a channel to receive events + eventCh := make(chan FileEvent, 10) // buffered channel to avoid blocking + + // create a file watcher for both files + watcher, err := NewFileWatcher(testFile1, func(event FileEvent) { + // send the event to the channel + select { + case eventCh <- event: + // event sent + default: + // channel full or closed, ignore + } + }) + require.NoError(t, err) + defer func() { _ = watcher.Close() }() + + // add the second file + err = watcher.AddPath(testFile2) + require.NoError(t, err) + + // remove the second file from the watcher + err = watcher.RemovePath(testFile2) + require.NoError(t, err) + + // modify the second file, but no event should be triggered + err = os.WriteFile(testFile2, []byte("modified file 2"), 0644) + require.NoError(t, err) + + // modify the first file to trigger an event + err = os.WriteFile(testFile1, []byte("modified file 1"), 0644) + require.NoError(t, err) + + // wait for the event to be received or timeout + select { + case event := <-eventCh: + assert.Equal(t, testFile1, event.Path, "Expected event for %s, got %s", testFile1, event.Path) + case <-time.After(5 * time.Second): + t.Fatal("Timeout waiting for file event") + } +} diff --git a/fileutils.go b/fileutils.go index 55be1b6..7743f85 100644 --- a/fileutils.go +++ b/fileutils.go @@ -2,10 +2,17 @@ package fileutils import ( + //nolint:gosec // Needed for compatibility + "crypto/md5" "crypto/rand" + //nolint:gosec // Needed for compatibility + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" "encoding/hex" "errors" "fmt" + "hash" "io" "os" "path/filepath" @@ -13,6 +20,26 @@ import ( "sort" "strings" "time" + + "github.com/go-pkgz/fileutils/enum" +) + +//go:generate enum -type=hashAlg -path=enum +//nolint:unused // This type is used by the enum generator +type hashAlg int + +// These constants are used by the enum generator +// +//nolint:unused // These constants are used by the enum generator +const ( + hashAlgMD5 hashAlg = iota + 1 + hashAlgSHA1 + hashAlgSHA256 + hashAlgSHA224 + hashAlgSHA384 + hashAlgSHA512 + hashAlgSHA512_224 + hashAlgSHA512_256 ) // IsFile returns true if filename exists @@ -38,7 +65,7 @@ func exists(name string, dir bool) bool { // CopyFile copies a file from source to dest, preserving mode. // Any existing file will be overwritten. -func CopyFile(src string, dst string) error { +func CopyFile(src, dst string) error { srcInfo, err := os.Stat(src) if err != nil { return fmt.Errorf("can't stat %s: %w", src, err) @@ -48,22 +75,22 @@ func CopyFile(src string, dst string) error { return fmt.Errorf("can't copy non-regular source file %s (%s)", src, srcInfo.Mode().String()) } - srcFh, err := os.Open(src) //nolint:gosec + srcFh, err := os.Open(src) //nolint:gosec // file path is provided by the caller if err != nil { return fmt.Errorf("can't open source file %s: %w", src, err) } - defer srcFh.Close() + defer func() { _ = srcFh.Close() }() - err = os.MkdirAll(filepath.Dir(dst), 0750) + err = os.MkdirAll(filepath.Dir(dst), 0o750) if err != nil { return fmt.Errorf("can't make destination directory %s: %w", filepath.Dir(dst), err) } - dstFh, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode()) //nolint:gosec + dstFh, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode()) //nolint:gosec // file path is provided by the caller if err != nil { return fmt.Errorf("can't create destination file %s: %w", dst, err) } - defer dstFh.Close() + defer func() { _ = dstFh.Close() }() size, err := io.Copy(dstFh, srcFh) if err != nil { @@ -77,7 +104,7 @@ func CopyFile(src string, dst string) error { } // CopyDir copies all files from src to dst, recursively -func CopyDir(src string, dst string) error { +func CopyDir(src, dst string) error { list, err := ListFiles(src) if err != nil { return fmt.Errorf("can't list source files in %s: %w", src, err) @@ -158,7 +185,7 @@ func SanitizePath(s string) string { s = strings.TrimSpace(s) s = reInvalidPathChars.ReplaceAllString(filepath.Clean(s), "_") - // Normalize path separators to '/' + // normalize path separators to '/' s = strings.ReplaceAll(s, `\`, "/") if len(s) > maxPathLength { @@ -251,10 +278,58 @@ func TouchFile(path string) error { if err != nil { return fmt.Errorf("failed to create file: %w", err) } - return f.Close() + if err := f.Close(); err != nil { + return fmt.Errorf("failed to close file: %w", err) + } + return nil } // file exists, update timestamps now := time.Now() return os.Chtimes(path, now, now) } + +// Checksum calculates the checksum of a file using the specified hash algorithm. +// Supported algorithms are MD5, SHA1, and SHA256. +func Checksum(path string, algo enum.HashAlg) (string, error) { + if path == "" { + return "", errors.New("empty path") + } + + var h hash.Hash + switch algo { + case enum.HashAlgMD5: + h = md5.New() //nolint:gosec // needed for compatibility + case enum.HashAlgSHA1: + h = sha1.New() //nolint:gosec // needed for compatibility + case enum.HashAlgSHA256: + h = sha256.New() + case enum.HashAlgSHA224: + h = sha256.New224() + case enum.HashAlgSHA384: + h = sha512.New384() + case enum.HashAlgSHA512: + h = sha512.New() + case enum.HashAlgSHA512_224: + h = sha512.New512_224() + case enum.HashAlgSHA512_256: + h = sha512.New512_256() + default: + return "", fmt.Errorf("unsupported hash algorithm: %v", algo) + } + + f, err := os.Open(path) //nolint:gosec // path is provided by the caller + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("file not found: %s", path) + } + return "", fmt.Errorf("failed to open file %s: %w", path, err) + } + defer func() { _ = f.Close() }() + + if _, err := io.Copy(h, f); err != nil { + return "", fmt.Errorf("failed to read file %s for hashing: %w", path, err) + } + + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/fileutils_test.go b/fileutils_test.go index ddd2037..987bc29 100644 --- a/fileutils_test.go +++ b/fileutils_test.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/go-pkgz/fileutils/enum" ) func TestExistsFile(t *testing.T) { @@ -148,11 +150,11 @@ func TestMoveFile(t *testing.T) { srcFile := filepath.Join(os.TempDir(), "move_test_src.txt") err := os.WriteFile(srcFile, []byte("test content"), 0600) require.NoError(t, err) - defer os.Remove(srcFile) + defer func() { _ = os.Remove(srcFile) }() // create temp destination dstFile := filepath.Join(os.TempDir(), "move_test_dst.txt") - defer os.Remove(dstFile) + defer func() { _ = os.Remove(dstFile) }() // perform move err = MoveFile(srcFile, dstFile) @@ -273,3 +275,93 @@ func TestTouchFile(t *testing.T) { require.Error(t, err) }) } +func TestChecksum(t *testing.T) { + // create a temporary test file + tmpDir := t.TempDir() + testFile := filepath.Join(tmpDir, "checksum_test.txt") + content := []byte("this is a test file for checksum calculation") + err := os.WriteFile(testFile, content, 0600) + require.NoError(t, err) + + // expected checksums (re-calculated) + expectedMD5 := "656b12fec36f7df11771b03c53e177ba" + expectedSHA1 := "be4c9cf3936f6d20ee0f38637605a405ea831168" + expectedSHA224 := "098477fd72b5128aa051cbd0a09010d14b3dd18114d66b061f0ff382" + expectedSHA384 := "a4de83d650a8a4d07483ad61296685d9d261e6edb940a025b8b981f90c17bdca794d45f202b2b8c3de5cd9c9bcf5e1e0" + expectedSHA512 := "a182278014c2a2d6d00de8442ab0e358e689269965eea6dfb7761abe019d0c34d47c181e19f1021901a5c0cf65b82871a0fa36b8bb187f9f5bf97ed182798e9e" + expectedSHA512_224 := "aab1745f6f1464c67c4a46d29c3a79132afe6d104f96a1266a69a278" + expectedSHA512_256 := "b5bc9721b180d5c79264f5fbb61404b516b6bfcb486c95b65329a1fe71ff6728" + expectedSHA256 := "7644ba794d6c4df31bd440ea9f7ecbcb8f2f3846cc58fcbf55d13560e168c863" + + t.Run("md5", func(t *testing.T) { + checksum, err := Checksum(testFile, enum.HashAlgMD5) + require.NoError(t, err) + assert.Equal(t, expectedMD5, checksum) + }) + + t.Run("sha1", func(t *testing.T) { + checksum, err := Checksum(testFile, enum.HashAlgSHA1) + require.NoError(t, err) + assert.Equal(t, expectedSHA1, checksum) + }) + + t.Run("sha256", func(t *testing.T) { + checksum, err := Checksum(testFile, enum.HashAlgSHA256) + require.NoError(t, err) + assert.Equal(t, expectedSHA256, checksum) + }) + + t.Run("sha224", func(t *testing.T) { + checksum, err := Checksum(testFile, enum.HashAlgSHA224) + require.NoError(t, err) + assert.Equal(t, expectedSHA224, checksum) + }) + + t.Run("sha384", func(t *testing.T) { + checksum, err := Checksum(testFile, enum.HashAlgSHA384) + require.NoError(t, err) + assert.Equal(t, expectedSHA384, checksum) + }) + + t.Run("sha512", func(t *testing.T) { + checksum, err := Checksum(testFile, enum.HashAlgSHA512) + require.NoError(t, err) + assert.Equal(t, expectedSHA512, checksum) + }) + + t.Run("sha512_224", func(t *testing.T) { + checksum, err := Checksum(testFile, enum.HashAlgSHA512_224) + require.NoError(t, err) + assert.Equal(t, expectedSHA512_224, checksum) + }) + + t.Run("sha512_256", func(t *testing.T) { + checksum, err := Checksum(testFile, enum.HashAlgSHA512_256) + require.NoError(t, err) + assert.Equal(t, expectedSHA512_256, checksum) + }) + + t.Run("parse from string", func(t *testing.T) { + // test parsing from string + alg, err := enum.ParseHashAlg("sha256") + require.NoError(t, err) + checksum, err := Checksum(testFile, alg) + require.NoError(t, err) + assert.Equal(t, expectedSHA256, checksum) + }) + + t.Run("errors", func(t *testing.T) { + _, err := Checksum("nonexistent.txt", enum.HashAlgMD5) + require.Error(t, err) + assert.Contains(t, err.Error(), "file not found") + + // test invalid algorithm parsing + _, err = enum.ParseHashAlg("unsupported_algo") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid hashAlg") + + _, err = Checksum("", enum.HashAlgMD5) + require.Error(t, err) + assert.Contains(t, err.Error(), "empty path") + }) +} diff --git a/go.mod b/go.mod index 5f9f128..351a75e 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,14 @@ module github.com/go-pkgz/fileutils go 1.19 -require github.com/stretchr/testify v1.8.1 +require ( + github.com/fsnotify/fsnotify v1.9.0 + github.com/stretchr/testify v1.8.1 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/sys v0.13.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2ec90f7..4b8579f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -10,6 +12,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 8b194ea8d5190d9d6ae0106c24c1ee4a36dcb48c Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 6 Apr 2025 02:58:59 -0500 Subject: [PATCH 2/6] Fix test race condition in TestWatchRecursive --- file_watcher_test.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/file_watcher_test.go b/file_watcher_test.go index 2c348d1..e13b7c7 100644 --- a/file_watcher_test.go +++ b/file_watcher_test.go @@ -98,17 +98,30 @@ func TestWatchRecursive(t *testing.T) { time.Sleep(300 * time.Millisecond) // clear channel from warmup events - select { - case <-eventCh: - // discard warmup event - default: - // no event yet, continue + // make sure we drain all events from the warmup + timeout := time.After(500 * time.Millisecond) + drainLoop := true + for drainLoop { + select { + case <-eventCh: + // discard warmup event + case <-timeout: + // no more events after timeout + drainLoop = false + default: + // if no events available but timeout hasn't occurred, + // wait a bit to avoid CPU spin + time.Sleep(50 * time.Millisecond) + } } // create a file in the subdirectory to trigger an event testFile := filepath.Join(subDir, "test.txt") err = os.WriteFile(testFile, []byte("test content"), 0644) require.NoError(t, err) + + // give a little time for the event to be processed + time.Sleep(100 * time.Millisecond) // wait for the event to be received or timeout From c2f8da8528003a35ce98876b655228e3bedf9400 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 6 Apr 2025 03:01:30 -0500 Subject: [PATCH 3/6] Allow coveralls step to fail without failing the build --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5154f2a..c22bdd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,5 +39,6 @@ jobs: - name: submit coverage run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov + continue-on-error: true env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f7040e5e4dbb91e275e527732805724fc247f390 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 6 Apr 2025 03:02:52 -0500 Subject: [PATCH 4/6] Replace if-else-if chain with switch statement --- file_watcher.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/file_watcher.go b/file_watcher.go index 038072b..c96c4ea 100644 --- a/file_watcher.go +++ b/file_watcher.go @@ -92,17 +92,19 @@ func (fw *FileWatcher) watch() { // convert fsnotify event to our event type var eventType enum.EventType - if event.Has(fsnotify.Create) { + + switch { + case event.Has(fsnotify.Create): eventType = enum.EventTypeCreate - } else if event.Has(fsnotify.Write) { + case event.Has(fsnotify.Write): eventType = enum.EventTypeWrite - } else if event.Has(fsnotify.Remove) { + case event.Has(fsnotify.Remove): eventType = enum.EventTypeRemove - } else if event.Has(fsnotify.Rename) { + case event.Has(fsnotify.Rename): eventType = enum.EventTypeRename - } else if event.Has(fsnotify.Chmod) { + case event.Has(fsnotify.Chmod): eventType = enum.EventTypeChmod - } else { + default: continue // unknown event type } From 029d2caaeb936ea77574bfe57b53f54b54459027 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 6 Apr 2025 03:04:13 -0500 Subject: [PATCH 5/6] Fix formatting issue --- file_watcher.go | 1 - 1 file changed, 1 deletion(-) diff --git a/file_watcher.go b/file_watcher.go index c96c4ea..04e9ed3 100644 --- a/file_watcher.go +++ b/file_watcher.go @@ -92,7 +92,6 @@ func (fw *FileWatcher) watch() { // convert fsnotify event to our event type var eventType enum.EventType - switch { case event.Has(fsnotify.Create): eventType = enum.EventTypeCreate From fba11ed2c5ab6f9d1a12c655cbe15c6a4f3f9c27 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 6 Apr 2025 03:09:43 -0500 Subject: [PATCH 6/6] Update Checksum function comment to list all supported hash algorithms --- fileutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fileutils.go b/fileutils.go index 7743f85..a02822d 100644 --- a/fileutils.go +++ b/fileutils.go @@ -290,7 +290,7 @@ func TouchFile(path string) error { } // Checksum calculates the checksum of a file using the specified hash algorithm. -// Supported algorithms are MD5, SHA1, and SHA256. +// Supported algorithms are MD5, SHA1, SHA224, SHA256, SHA384, SHA512, SHA512_224, and SHA512_256. func Checksum(path string, algo enum.HashAlg) (string, error) { if path == "" { return "", errors.New("empty path")