Skip to content

Commit ea95ec0

Browse files
umputunumputun
committed
refactor(dedup): unify DeDup algorithms into single stable implementation
replace unstable O(n²) DeDup with stable O(n) map-based algorithm that preserves order of first occurrences. DeDupBig is now deprecated and simply calls DeDup for backwards compatibility. this change eliminates the confusing dual-algorithm design, improves performance for all slice sizes, and ensures predictable behavior. the new implementation no longer mutates input slices, returning a new slice instead. verified by comprehensive tests including stability checks and non-mutation verification. benchmarks show significant performance improvement for small slices with no degradation for large ones.
1 parent 94f699b commit ea95ec0

File tree

2 files changed

+131
-25
lines changed

2 files changed

+131
-25
lines changed

‎stringutils.go‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -31,31 +31,13 @@ func ContainsAnySubstring(s string, subStrings []string) bool {
3131
return false
3232
}
3333

34-
// DeDup remove duplicates from slice. optimized for performance, good for short slices only!
34+
// DeDup remove duplicates from slice.
35+
// This function is stable - it preserves the order of first occurrences.
3536
func DeDup(keys []string) []string {
3637
if len(keys) == 0 {
3738
return nil
3839
}
39-
l := len(keys) - 1
40-
for i := 0; i < l; i++ {
41-
for j := i + 1; j <= l; j++ {
42-
if keys[i] == keys[j] {
43-
keys[j] = keys[l]
44-
keys = keys[0:l]
45-
l--
46-
j--
47-
}
48-
}
49-
}
50-
return keys
51-
}
52-
53-
// DeDupBig remove duplicates from slice. Should be used instead of DeDup for large slices
54-
func DeDupBig(keys []string) (result []string) {
55-
if len(keys) == 0 {
56-
return nil
57-
}
58-
result = make([]string, 0, len(keys))
40+
result := make([]string, 0, len(keys))
5941
visited := make(map[string]struct{}, len(keys))
6042
for _, k := range keys {
6143
if _, found := visited[k]; !found {
@@ -66,6 +48,12 @@ func DeDupBig(keys []string) (result []string) {
6648
return result
6749
}
6850

51+
// DeDupBig remove duplicates from slice.
52+
// Deprecated: Use DeDup instead. This function now just calls DeDup for backwards compatibility.
53+
func DeDupBig(keys []string) []string {
54+
return DeDup(keys)
55+
}
56+
6957
// SliceToString converts slice of any to slice of string
7058
func SliceToString(s []any) []string {
7159
if len(s) == 0 {

‎stringutils_test.go‎

Lines changed: 122 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,20 @@ func TestContains(t *testing.T) {
1616
}{
1717
{"finds string", "test", []string{"test", "example"}, true},
1818
{"doesn't find string", "missing", []string{"test", "example"}, false},
19+
{"nil slice", "test", nil, false},
20+
{"empty slice", "test", []string{}, false},
21+
{"empty string source", "", []string{"test", "example"}, false},
22+
{"empty string in slice", "test", []string{"", "test", "example"}, true},
23+
{"empty string match", "", []string{"", "test"}, true},
24+
{"unicode string", "тест", []string{"test", "тест", "example"}, true},
25+
{"unicode not found", "тест", []string{"test", "example"}, false},
26+
{"special characters", "test@#$", []string{"test@#$", "example"}, true},
27+
{"case sensitive", "Test", []string{"test", "example"}, false},
28+
{"duplicates in slice", "test", []string{"test", "test", "test"}, true},
29+
{"single item slice found", "test", []string{"test"}, true},
30+
{"single item slice not found", "test", []string{"example"}, false},
31+
{"spaces in string", "test string", []string{"test string", "example"}, true},
32+
{"partial match should not find", "test", []string{"testing", "testable"}, false},
1933
}
2034

2135
for _, tt := range tests {
@@ -38,6 +52,22 @@ func TestContainsAnySubstring(t *testing.T) {
3852
{"empty substring with match", "hello world", []string{"", "world"}, true},
3953
{"only empty substring", "hello world", []string{""}, false},
4054
{"multiple empty substrings", "hello world", []string{"", "", ""}, false},
55+
{"nil slice", "hello world", nil, false},
56+
{"empty slice", "hello world", []string{}, false},
57+
{"empty main string", "", []string{"test", "example"}, false},
58+
{"empty main string with empty substring", "", []string{""}, false},
59+
{"case sensitive", "Hello World", []string{"hello", "WORLD"}, false},
60+
{"case sensitive match", "Hello World", []string{"Hello", "missing"}, true},
61+
{"unicode substring", "привет мир", []string{"мир", "test"}, true},
62+
{"unicode not found", "привет мир", []string{"hello", "world"}, false},
63+
{"special characters", "test@#$%^&*()", []string{"@#$", "missing"}, true},
64+
{"overlapping matches", "testing", []string{"test", "sting", "ing"}, true},
65+
{"partial word match", "testing", []string{"test"}, true},
66+
{"multiple matches", "hello world", []string{"hello", "world", "test"}, true},
67+
{"very long string", strings.Repeat("a", 1000) + "needle" + strings.Repeat("b", 1000), []string{"needle"}, true},
68+
{"single character match", "hello", []string{"h", "x"}, true},
69+
{"newline in string", "hello\nworld", []string{"\n"}, true},
70+
{"tab in string", "hello\tworld", []string{"\t"}, true},
4171
}
4272

4373
for _, tt := range tests {
@@ -57,35 +87,68 @@ func TestDeDup(t *testing.T) {
5787
{"empty input", []string{}, nil},
5888
{"removes duplicates", []string{"test", "test", "example"}, []string{"test", "example"}},
5989
{"no duplicates", []string{"test", "test2", "example"}, []string{"test", "test2", "example"}},
90+
{"all duplicates", []string{"same", "same", "same", "same"}, []string{"same"}},
91+
{"alternating duplicates", []string{"a", "b", "a", "b", "a"}, []string{"a", "b"}},
92+
{"many duplicates of one", []string{"x", "x", "x", "y", "x", "x"}, []string{"x", "y"}},
93+
{"single element", []string{"alone"}, []string{"alone"}},
94+
{"unicode strings", []string{"тест", "тест", "мир", "тест"}, []string{"тест", "мир"}},
95+
{"strings with spaces", []string{"hello world", "hello world", "test"}, []string{"hello world", "test"}},
96+
{"special characters", []string{"@#$", "^&*", "@#$", "^&*"}, []string{"@#$", "^&*"}},
97+
{"empty strings", []string{"", "", "test", ""}, []string{"", "test"}},
98+
{"case sensitive", []string{"Test", "test", "Test"}, []string{"Test", "test"}},
99+
{"adjacent duplicates", []string{"a", "a", "b", "b", "c", "c"}, []string{"a", "b", "c"}},
100+
{"scattered duplicates", []string{"a", "b", "c", "a", "d", "b"}, []string{"a", "b", "c", "d"}},
60101
}
61102

62103
for _, tt := range tests {
63104
t.Run(tt.name, func(t *testing.T) {
64-
assert.Equal(t, tt.want, DeDup(tt.keys))
105+
// make a copy to verify no mutation
106+
var original []string
107+
if tt.keys != nil {
108+
original = make([]string, len(tt.keys))
109+
copy(original, tt.keys)
110+
}
111+
112+
result := DeDup(tt.keys)
113+
assert.Equal(t, tt.want, result)
114+
115+
// verify that original is not mutated
116+
if tt.keys != nil {
117+
assert.Equal(t, original, tt.keys, "should not mutate original slice")
118+
}
65119
})
66120
}
67121
}
68122

69123
func TestDeDupBig(t *testing.T) {
124+
// DeDupBig is deprecated and just calls DeDup, so we only need basic tests for backwards compatibility
70125
tests := []struct {
71126
name string
72127
keys []string
73128
want []string
74129
}{
75130
{"nil input", nil, nil},
76-
{"empty input", []string{}, nil},
77131
{"removes duplicates", []string{"test", "test", "example"}, []string{"test", "example"}},
78-
{"no duplicates", []string{"test", "test2", "example"}, []string{"test", "test2", "example"}},
132+
{"verify stability", []string{"a", "b", "c", "b", "d", "a", "e"}, []string{"a", "b", "c", "d", "e"}},
79133
}
80134

81135
for _, tt := range tests {
82136
t.Run(tt.name, func(t *testing.T) {
83-
assert.Equal(t, tt.want, DeDupBig(tt.keys))
137+
result := DeDupBig(tt.keys)
138+
assert.Equal(t, tt.want, result)
139+
140+
// verify it returns same result as DeDup
141+
assert.Equal(t, DeDup(tt.keys), result, "DeDupBig should return same result as DeDup")
84142
})
85143
}
86144
}
87145

88146
func TestSliceToString(t *testing.T) {
147+
type testStruct struct {
148+
Name string
149+
Age int
150+
}
151+
89152
tests := []struct {
90153
name string
91154
in []interface{}
@@ -96,6 +159,17 @@ func TestSliceToString(t *testing.T) {
96159
{"converts number to string", []any{1, 2, 3}, []string{"1", "2", "3"}},
97160
{"converts mixed slice to string", []any{1, "aaa", true, 0.55}, []string{"1", "aaa", "true", "0.55"}},
98161
{"converts slice of byte slices to string", []any{[]byte("hi"), []byte("there")}, []string{"hi", "there"}},
162+
{"nil values", []any{nil, "test", nil}, []string{"<nil>", "test", "<nil>"}},
163+
{"empty byte slice", []any{[]byte{}, []byte("test")}, []string{"", "test"}},
164+
{"byte slice with null bytes", []any{[]byte{0x00, 0x01, 0x02}}, []string{"\x00\x01\x02"}},
165+
{"complex types", []any{testStruct{Name: "John", Age: 30}, map[string]int{"a": 1}}, []string{"{John 30}", "map[a:1]"}},
166+
{"pointers", []any{&testStruct{Name: "Jane", Age: 25}}, []string{"&{Jane 25}"}},
167+
{"arrays", []any{[3]int{1, 2, 3}}, []string{"[1 2 3]"}},
168+
{"slices", []any{[]int{4, 5, 6}}, []string{"[4 5 6]"}},
169+
{"negative numbers", []any{-1, -999, -0.5}, []string{"-1", "-999", "-0.5"}},
170+
{"large numbers", []any{int64(9223372036854775807)}, []string{"9223372036854775807"}},
171+
{"unicode in byte slice", []any{[]byte("привет мир")}, []string{"привет мир"}},
172+
{"special chars in byte slice", []any{[]byte("@#$%^&*()")}, []string{"@#$%^&*()"}},
99173
}
100174

101175
for _, tt := range tests {
@@ -117,6 +191,15 @@ func TestHasCommonElement(t *testing.T) {
117191
{"one slice is empty", []string{}, []string{"x", "y", "z", "w"}, false},
118192
{"element found at the start", []string{"a", "b", "c", "d"}, []string{"a", "x", "y", "z"}, true},
119193
{"element found at the end", []string{"a", "b", "c", "d"}, []string{"x", "y", "z", "d"}, true},
194+
{"same slice twice", []string{"a", "b", "c"}, []string{"a", "b", "c"}, true},
195+
{"same slice twice empty", []string{}, []string{}, false},
196+
{"single element both", []string{"x"}, []string{"x"}, true},
197+
{"single element no match", []string{"x"}, []string{"y"}, false},
198+
{"all common", []string{"a", "b", "c"}, []string{"c", "b", "a"}, true},
199+
{"unicode elements", []string{"тест", "мир"}, []string{"привет", "мир"}, true},
200+
{"case sensitive", []string{"Test", "test"}, []string{"TEST", "test"}, true},
201+
{"nil first slice", nil, []string{"a"}, false},
202+
{"nil second slice", []string{"a"}, nil, false},
120203
}
121204

122205
for _, tc := range tests {
@@ -413,6 +496,41 @@ func TestRandomWord(t *testing.T) {
413496
}
414497
}
415498

499+
func BenchmarkDeDup(b *testing.B) {
500+
// small slice with duplicates
501+
small := []string{"a", "b", "a", "c", "b", "d", "e", "a"}
502+
503+
b.Run("small", func(b *testing.B) {
504+
for i := 0; i < b.N; i++ {
505+
_ = DeDup(small)
506+
}
507+
})
508+
509+
// medium slice
510+
medium := make([]string, 100)
511+
for i := 0; i < 100; i++ {
512+
medium[i] = string(rune('a' + (i % 10))) // only 10 unique values
513+
}
514+
515+
b.Run("medium", func(b *testing.B) {
516+
for i := 0; i < b.N; i++ {
517+
_ = DeDup(medium)
518+
}
519+
})
520+
521+
// large slice with many duplicates
522+
large := make([]string, 1000)
523+
for i := 0; i < 1000; i++ {
524+
large[i] = string(rune('a' + (i % 26))) // only 26 unique values
525+
}
526+
527+
b.Run("large", func(b *testing.B) {
528+
for i := 0; i < b.N; i++ {
529+
_ = DeDup(large)
530+
}
531+
})
532+
}
533+
416534
func BenchmarkSliceToString(b *testing.B) {
417535
tmpl := []any{[]byte("fdjndfg")}
418536
b.Run("small slice", func(b *testing.B) {

0 commit comments

Comments
 (0)