diff --git a/server/rates.go b/server/rates.go
index 5677c47..5dd3824 100644
--- a/server/rates.go
+++ b/server/rates.go
@@ -49,7 +49,11 @@ func (rl *RateLimiter) Inc(key string) (uint32, func(), func()) {
 		}
 
 		rl.total.Add(MinusOne)
-		val.Add(MinusOne)
+
+		if val.Add(MinusOne) == 0 {
+			// potential race, but ok
+			rl.Map.Delete(key)
+		}
 	}
 
 	return new, pass, fail