Skip to content

Commit 60595ac

Browse files
committed
refactor: tidy codebase — fix bugs, remove duplication
- Fix broken regex in detectors/base.py: r"\\D+" → r"\D+" (digits_only was a no-op) - Remove duplicate dataclass/typing imports and duplicate Detector Protocol definition in base.py - Remove duplicate `phonenumbers` and `stdnum` imports in regexes.py - Remove duplicate `shannon_entropy` function in entropy.py (use utils.py import) - Remove unused `Detector` import and `math` import in entropy.py - Fix syntax error in decors.py: '. . .' → '...' - Replace wildcard import in hide.py with explicit named import - Fix transforms/tokenise.py which was an identical copy of mask.py; implement SHA-256 tokenization - Remove duplicate Python SDK section in README.md - Replace placeholder description in pyproject.toml https://claude.ai/code/session_01SmfiwftPFKGgQr1W4uYYjs
1 parent 40dcf8d commit 60595ac

8 files changed

Lines changed: 57 additions & 87 deletions

File tree

‎README.md‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -57,18 +57,6 @@ redactable --policy gdpr.yaml input.log output.redacted.log
5757
````
5858

5959

60-
### Python SDK
61-
62-
````bash
63-
from redactable import apply
64-
65-
data = "Customer email: test@example.com"
66-
result = apply(data, policy="gdpr.yaml")
67-
print(result)
68-
# → "Customer email: ****@example.com"
69-
````
70-
71-
7260
### Python SDK
7361

7462
````bash

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
44

55
[project]
66
name = "redactable"
7-
description = "One-line value prop."
7+
description = "Policy-driven framework for data redaction, masking, and privacy-preserving transformations."
88
readme = "README.md"
99
requires-python = ">=3.13"
1010
license = { text = "MIT" }

‎src/redactable/decors.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
def redactable_io(policy): . . .
1+
def redactable_io(policy): ...

‎src/redactable/detectors/base.py‎

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,55 +3,58 @@
33
44
Contents:
55
- Finding: dataclass representing a detected entity.
6-
- Detector: Protocol interface that all detectors must implement.
6+
- Match: dataclass for the module-level detection system.
7+
- Detector: Protocol interface that Finding-based detectors must implement.
78
- Shared helper functions: digits_only, luhn_ok, guess_card_brand.
89
"""
910

10-
from dataclasses import dataclass
11-
from typing import Iterable, Optional, Protocol, Tuple, Dict, Any
11+
from __future__ import annotations
12+
1213
import re
14+
from dataclasses import dataclass
15+
from typing import Any, Iterable, Optional, Protocol
1316

1417
# --------------------------------------------------------------------
15-
# Shared type aliases
16-
Span = Tuple[int, int]
17-
Extras = Dict[str, Any]
18+
# Match-based detection types (used by individual detector modules)
1819

1920

20-
from dataclasses import dataclass
21-
from typing import Iterable, Protocol, TypedDict, Optional, Dict, Any, List
22-
2321
@dataclass(slots=True)
2422
class Match:
2523
label: str # e.g. "EMAIL", "CREDIT_CARD"
2624
start: int # byte/char index in the input text
2725
end: int
2826
value: str # matched text (pre-transform)
2927
confidence: float = 1.0 # 0..1
30-
meta: dict[str, Any] = None
28+
meta: dict[str, Any] | None = None
3129

32-
class Detector(Protocol):
33-
name: str
34-
labels: tuple[str, ...]
35-
def detect(self, text: str, *, context: Optional[dict[str, Any]] = None) -> Iterable[Match]: ...
3630

3731
# Simple registry
38-
_REGISTRY: Dict[str, Detector] = {}
39-
_LABEL_TO_DETECTORS: Dict[str, List[str]] = {}
32+
_REGISTRY: dict[str, Any] = {}
33+
_LABEL_TO_DETECTORS: dict[str, list[str]] = {}
34+
4035

41-
def register(detector: Detector) -> None:
36+
def register(detector: Any) -> None:
4237
_REGISTRY[detector.name] = detector
4338
for label in detector.labels:
4439
_LABEL_TO_DETECTORS.setdefault(label, []).append(detector.name)
4540

46-
def get(name: str) -> Detector:
41+
42+
def get(name: str) -> Any:
4743
return _REGISTRY[name]
4844

49-
def detectors_for(label: str) -> list[Detector]:
50-
return [ _REGISTRY[n] for n in _LABEL_TO_DETECTORS.get(label, []) ]
5145

52-
def all_detectors() -> list[Detector]:
46+
def detectors_for(label: str) -> list[Any]:
47+
return [_REGISTRY[n] for n in _LABEL_TO_DETECTORS.get(label, [])]
48+
49+
50+
def all_detectors() -> list[Any]:
5351
return list(_REGISTRY.values())
5452

53+
54+
# --------------------------------------------------------------------
55+
# Finding-based detection types (used by DetectorRegistry)
56+
57+
5558
@dataclass(slots=True)
5659
class Finding:
5760
"""
@@ -65,12 +68,13 @@ class Finding:
6568
normalized: Canonicalized form (e.g. digits-only phone number).
6669
extras: Additional metadata (brand, region, reasons, etc.).
6770
"""
71+
6872
kind: str
6973
value: str
70-
span: Span
74+
span: tuple[int, int]
7175
confidence: float
7276
normalized: Optional[str] = None
73-
extras: Extras | None = None
77+
extras: dict[str, Any] | None = None
7478

7579
def __post_init__(self) -> None:
7680
if not (0.0 <= self.confidence <= 1.0):
@@ -84,22 +88,26 @@ def __str__(self) -> str:
8488

8589
class Detector(Protocol):
8690
"""
87-
Protocol that all detectors must follow.
91+
Protocol that Finding-based detectors must follow.
8892
Each detector must expose a `name` and a `detect` method.
8993
"""
94+
9095
name: str
9196

9297
def detect(self, text: str) -> Iterable[Finding]: ...
9398

99+
94100
# --------------------------------------------------------------------
95101
# Shared helpers
96102

97-
_DIGITS = re.compile(r"\\D+")
103+
_DIGITS = re.compile(r"\D+")
104+
98105

99106
def digits_only(s: str) -> str:
100107
"""Strip all non-digit characters from a string."""
101108
return _DIGITS.sub("", s)
102109

110+
103111
def luhn_ok(num: str) -> bool:
104112
"""
105113
Check if a string of digits passes the Luhn algorithm.
@@ -120,6 +128,7 @@ def luhn_ok(num: str) -> bool:
120128
alt = not alt
121129
return total % 10 == 0
122130

131+
123132
def guess_card_brand(pan: str) -> str | None:
124133
"""
125134
Make a naive guess of card brand from PAN digits.
@@ -154,5 +163,3 @@ def guess_card_brand(pan: str) -> str | None:
154163
return "unionpay"
155164

156165
return None
157-
158-

‎src/redactable/detectors/entropy.py‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12,25 +12,10 @@
1212
"""
1313

1414
import re
15-
from .base import Match, register, Finding, Detector
16-
from .utils import shannon_entropy, looks_like_secret
1715
from typing import Iterable
18-
import math
1916

20-
21-
# --------------------------------------------------------------------
22-
# Helpers
23-
24-
def shannon_entropy(s: str) -> float:
25-
"""
26-
Calculate Shannon entropy of a string.
27-
Returns a value >= 0, higher means more random.
28-
"""
29-
if not s:
30-
return 0.0
31-
freq = {ch: s.count(ch) for ch in set(s)}
32-
n = len(s)
33-
return -sum((c/n) * math.log2(c/n) for c in freq.values())
17+
from .base import Finding, Match, register
18+
from .utils import looks_like_secret, shannon_entropy
3419

3520
# --------------------------------------------------------------------
3621
# Regex pattern: matches candidate secrets
@@ -56,6 +41,7 @@ class HighEntropyTokenDetector:
5641
entropy_threshold: minimum Shannon entropy (default ~3.5).
5742
min_len: minimum string length to consider.
5843
"""
44+
5945
name = "high_entropy_token"
6046

6147
def __init__(self, entropy_threshold: float = 3.5, min_len: int = 24) -> None:
@@ -86,6 +72,7 @@ def detect(self, text: str) -> Iterable[Finding]:
8672
# Tokens separated by non-word; allow -,_,= typical in JWT/base64url
8773
_TOKEN = re.compile(r'([A-Za-z0-9_\-=+/]{20,})')
8874

75+
8976
class EntropyDetector:
9077
name = "entropy"
9178
labels = ("SECRET",)
@@ -101,6 +88,7 @@ def detect(self, text: str, *, context=None):
10188
continue
10289
H = shannon_entropy(token)
10390
if H >= threshold:
104-
yield Match("SECRET", m.start(1), m.end(1), token, min(0.99, 0.7 + (H-threshold)/4), {"entropy": H})
91+
yield Match("SECRET", m.start(1), m.end(1), token, min(0.99, 0.7 + (H - threshold) / 4), {"entropy": H})
92+
10593

106-
register(EntropyDetector())
94+
register(EntropyDetector())

‎src/redactable/detectors/regexes.py‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
"""
1818

1919
import re
20-
from typing import Iterable, Dict, Any
20+
from typing import Any, Iterable
2121

22-
from .base import Finding, Detector, digits_only, luhn_ok, guess_card_brand
22+
from .base import Finding, digits_only, luhn_ok, guess_card_brand
2323

2424
# --------------------------------------------------------------------
2525
# Optional external dependencies (gracefully degrade if missing)
@@ -86,11 +86,6 @@ def detect(self, text: str) -> Iterable[Finding]:
8686
extras={"luhn_valid": ok, "brand": brand},
8787
)
8888

89-
try:
90-
import phonenumbers # type: ignore
91-
except Exception: # pragma: no cover
92-
phonenumbers = None
93-
9489
# --------------------------------------------------------------------
9590
# Simple phone regex fallback
9691
RE_PHONE = re.compile(
@@ -157,7 +152,7 @@ def detect(self, text: str) -> Iterable[Finding]:
157152
start, end = m.span()
158153
conf = 0.6
159154
norm = raw
160-
extras: Dict[str, Any] = {}
155+
extras: dict[str, Any] = {}
161156
# If email-validator is available, upgrade confidence
162157
if validate_email is not None:
163158
try:
@@ -176,13 +171,6 @@ def detect(self, text: str) -> Iterable[Finding]:
176171
extras=extras,
177172
)
178173

179-
try:
180-
from stdnum import iban as std_iban # type: ignore
181-
from stdnum.gb import nhs as std_nhs # type: ignore
182-
from stdnum.us import ssn as std_us_ssn # type: ignore
183-
except Exception: # pragma: no cover
184-
std_iban = std_nhs = std_us_ssn = None
185-
186174
# --------------------------------------------------------------------
187175
# Regex patterns
188176
RE_NHS = re.compile(r"\b(\d{3})[\s-]?(\d{3})[\s-]?(\d{4})\b")

‎src/redactable/hide.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,3 @@
1-
"""
2-
Use this as a main entrypoint?
3-
"""
1+
from .decors import redactable_io
42

5-
from .decors import *
6-
7-
__all__ = []
3+
__all__ = ["redactable_io"]
Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
1+
import hashlib
12
from typing import Iterable
3+
24
from redactable.detectors import Finding
35

4-
def _mask(value: str, keep_head: int = 0, keep_tail: int = 4, glyph: str = "•") -> str:
5-
if len(value) <= keep_head + keep_tail:
6-
return glyph * len(value)
7-
return value[:keep_head] + glyph * (len(value) - keep_head - keep_tail) + value[-keep_tail:]
86

9-
def mask_in_place(text: str, findings: Iterable[Finding], keep_head: int = 0, keep_tail: int = 4, glyph: str = "•") -> str:
7+
def _sha256(value: str, salt: str = "") -> str:
8+
return hashlib.sha256((salt + value).encode("utf-8")).hexdigest()
9+
10+
11+
def tokenise_in_place(text: str, findings: Iterable[Finding], salt: str = "") -> str:
1012
out = text
1113
for f in sorted(findings, key=lambda x: x.span[0], reverse=True):
1214
s, e = f.span
13-
out = out[:s] + _mask(out[s:e], keep_head, keep_tail, glyph) + out[e:]
15+
token = _sha256(f.normalized or f.value, salt)
16+
out = out[:s] + token + out[e:]
1417
return out

0 commit comments

Comments
 (0)