34 lines
543 B
Go
34 lines
543 B
Go
|
package ingestion
|
||
|
|
|
||
|
|
import "sync"
|
||
|
|
|
||
|
|
type Deduplicator struct {
|
||
|
|
mu sync.RWMutex
|
||
|
|
seen map[string]struct{}
|
||
|
|
}
|
||
|
|
|
||
|
|
func NewDeduplicator() *Deduplicator {
|
||
|
|
return &Deduplicator{
|
||
|
|
seen: make(map[string]struct{}),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (d *Deduplicator) Exists(key string) bool {
|
||
|
|
d.mu.RLock()
|
||
|
|
defer d.mu.RUnlock()
|
||
|
|
_, ok := d.seen[key]
|
||
|
|
return ok
|
||
|
|
}
|
||
|
|
|
||
|
|
func (d *Deduplicator) Mark(key string) {
|
||
|
|
d.mu.Lock()
|
||
|
|
defer d.mu.Unlock()
|
||
|
|
d.seen[key] = struct{}{}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (d *Deduplicator) Reset() {
|
||
|
|
d.mu.Lock()
|
||
|
|
defer d.mu.Unlock()
|
||
|
|
d.seen = make(map[string]struct{})
|
||
|
|
}
|