mirror of
https://github.com/hanzoai/email-verifier.git
synced 2026-10-30 15:04:29 +00:00
812 lines
21 KiB
Go
812 lines
21 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"database/sql"
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
emailverifier "github.com/hanzoai/email-verifier"
|
|
_ "github.com/mattn/go-sqlite3"
|
|
)
|
|
|
|
type VerificationStatus string
|
|
|
|
const (
|
|
StatusClean VerificationStatus = "clean"
|
|
StatusBounced VerificationStatus = "bounced"
|
|
StatusBadSyntax VerificationStatus = "bad_syntax"
|
|
StatusBadNoMX VerificationStatus = "bad_no_mx"
|
|
StatusDisposable VerificationStatus = "bad_disposable"
|
|
StatusRoleAccount VerificationStatus = "role_account"
|
|
)
|
|
|
|
type VerifyResult struct {
|
|
Email string
|
|
Status VerificationStatus
|
|
Reason string
|
|
Domain string
|
|
HasMX bool
|
|
Disposable bool
|
|
Role bool
|
|
Free bool
|
|
Suggestion string
|
|
}
|
|
|
|
type DomainCacheEntry struct {
|
|
HasMX bool
|
|
Err error
|
|
}
|
|
|
|
type BatchVerifier struct {
|
|
verifier *emailverifier.Verifier
|
|
blacklist map[string]struct{}
|
|
domainCache sync.Map
|
|
concurrency int
|
|
}
|
|
|
|
func NewBatchVerifier(blacklistPath string, concurrency int) (*BatchVerifier, error) {
|
|
v := emailverifier.NewVerifier().EnableDomainSuggest()
|
|
bl := make(map[string]struct{})
|
|
|
|
if blacklistPath != "" {
|
|
if f, err := os.Open(blacklistPath); err == nil {
|
|
defer f.Close()
|
|
if strings.HasSuffix(blacklistPath, ".json") {
|
|
var list []string
|
|
if err := json.NewDecoder(f).Decode(&list); err == nil {
|
|
for _, em := range list {
|
|
bl[strings.ToLower(strings.TrimSpace(em))] = struct{}{}
|
|
}
|
|
}
|
|
} else {
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
em := strings.ToLower(strings.TrimSpace(scanner.Text()))
|
|
if strings.Contains(em, "@") {
|
|
bl[em] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
fmt.Printf("Loaded %d blacklisted/bounced emails into verifier memory.\n", len(bl))
|
|
}
|
|
}
|
|
|
|
return &BatchVerifier{
|
|
verifier: v,
|
|
blacklist: bl,
|
|
concurrency: concurrency,
|
|
}, nil
|
|
}
|
|
|
|
func (bv *BatchVerifier) checkDomainMX(domain string) (bool, error) {
|
|
if val, ok := bv.domainCache.Load(domain); ok {
|
|
entry := val.(DomainCacheEntry)
|
|
return entry.HasMX, entry.Err
|
|
}
|
|
|
|
mx, err := bv.verifier.CheckMX(domain)
|
|
hasMX := false
|
|
if mx != nil && mx.HasMXRecord {
|
|
hasMX = true
|
|
}
|
|
bv.domainCache.Store(domain, DomainCacheEntry{HasMX: hasMX, Err: err})
|
|
return hasMX, err
|
|
}
|
|
|
|
func (bv *BatchVerifier) VerifyOne(rawEmail string) VerifyResult {
|
|
email := strings.ToLower(strings.TrimSpace(rawEmail))
|
|
res := VerifyResult{
|
|
Email: email,
|
|
Status: StatusClean,
|
|
}
|
|
|
|
if email == "" {
|
|
res.Status = StatusBadSyntax
|
|
res.Reason = "empty_email"
|
|
return res
|
|
}
|
|
|
|
// 1. Check Blacklist / Prior Bounces
|
|
if _, ok := bv.blacklist[email]; ok {
|
|
res.Status = StatusBounced
|
|
res.Reason = "blacklisted_or_previously_bounced"
|
|
return res
|
|
}
|
|
|
|
// 2. Syntax Check
|
|
syntax := bv.verifier.ParseAddress(email)
|
|
if !syntax.Valid {
|
|
res.Status = StatusBadSyntax
|
|
res.Reason = "invalid_rfc5322_syntax"
|
|
return res
|
|
}
|
|
|
|
res.Domain = syntax.Domain
|
|
res.Free = bv.verifier.IsFreeDomain(syntax.Domain)
|
|
res.Role = bv.verifier.IsRoleAccount(syntax.Username)
|
|
res.Disposable = bv.verifier.IsDisposable(syntax.Domain)
|
|
res.Suggestion = bv.verifier.SuggestDomain(syntax.Domain)
|
|
|
|
// 3. Disposable Domain Check
|
|
if res.Disposable {
|
|
res.Status = StatusDisposable
|
|
res.Reason = "disposable_domain"
|
|
return res
|
|
}
|
|
|
|
// 4. DNS MX Check (Cached per domain)
|
|
hasMX, err := bv.checkDomainMX(syntax.Domain)
|
|
res.HasMX = hasMX
|
|
if !hasMX || err != nil {
|
|
res.Status = StatusBadNoMX
|
|
if err != nil {
|
|
res.Reason = err.Error()
|
|
} else {
|
|
res.Reason = "no_mx_records_found"
|
|
}
|
|
return res
|
|
}
|
|
|
|
// 5. Role Account Check
|
|
if res.Role {
|
|
res.Status = StatusRoleAccount
|
|
res.Reason = "role_based_mailbox"
|
|
return res
|
|
}
|
|
|
|
// 6. Clean
|
|
res.Status = StatusClean
|
|
res.Reason = "verified_clean_active_mx"
|
|
return res
|
|
}
|
|
|
|
type APIVerifyResponse struct {
|
|
Email string `json:"email"`
|
|
Deliverable bool `json:"deliverable"`
|
|
Status string `json:"status"`
|
|
Reason string `json:"reason"`
|
|
Domain string `json:"domain"`
|
|
HasMX bool `json:"has_mx"`
|
|
Disposable bool `json:"disposable"`
|
|
Role bool `json:"role"`
|
|
Free bool `json:"free"`
|
|
Suggestion string `json:"suggestion,omitempty"`
|
|
}
|
|
|
|
func startAPIServer(bv *BatchVerifier, port int) {
|
|
mux := http.NewServeMux()
|
|
|
|
mux.HandleFunc("/v1/email/verify", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Header().Set("Access-Control-Allow-Origin", "*")
|
|
var emails []string
|
|
|
|
if r.Method == http.MethodGet {
|
|
em := r.URL.Query().Get("email")
|
|
if em != "" {
|
|
emails = append(emails, em)
|
|
}
|
|
} else if r.Method == http.MethodPost {
|
|
var body struct {
|
|
Email string `json:"email"`
|
|
Emails []string `json:"emails"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err == nil {
|
|
if body.Email != "" {
|
|
emails = append(emails, body.Email)
|
|
}
|
|
if len(body.Emails) > 0 {
|
|
emails = append(emails, body.Emails...)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(emails) == 0 {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"error": "missing 'email' query param or JSON body"})
|
|
return
|
|
}
|
|
|
|
if len(emails) == 1 {
|
|
res := bv.VerifyOne(emails[0])
|
|
resp := APIVerifyResponse{
|
|
Email: res.Email,
|
|
Deliverable: res.Status == StatusClean,
|
|
Status: string(res.Status),
|
|
Reason: res.Reason,
|
|
Domain: res.Domain,
|
|
HasMX: res.HasMX,
|
|
Disposable: res.Disposable,
|
|
Role: res.Role,
|
|
Free: res.Free,
|
|
Suggestion: res.Suggestion,
|
|
}
|
|
_ = json.NewEncoder(w).Encode(resp)
|
|
return
|
|
}
|
|
|
|
var results []APIVerifyResponse
|
|
for _, em := range emails {
|
|
res := bv.VerifyOne(em)
|
|
results = append(results, APIVerifyResponse{
|
|
Email: res.Email,
|
|
Deliverable: res.Status == StatusClean,
|
|
Status: string(res.Status),
|
|
Reason: res.Reason,
|
|
Domain: res.Domain,
|
|
HasMX: res.HasMX,
|
|
Disposable: res.Disposable,
|
|
Role: res.Role,
|
|
Free: res.Free,
|
|
Suggestion: res.Suggestion,
|
|
})
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{"data": results, "count": len(results)})
|
|
})
|
|
|
|
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"status": "ok",
|
|
"suppressions": len(bv.blacklist),
|
|
"service": "hanzo-email-verifier",
|
|
})
|
|
})
|
|
|
|
addr := fmt.Sprintf("127.0.0.1:%d", port)
|
|
fmt.Printf("🚀 Email Verifier API microservice listening on http://%s/v1/email/verify\n", addr)
|
|
srv := &http.Server{
|
|
Addr: addr,
|
|
Handler: mux,
|
|
ReadTimeout: 10 * time.Second,
|
|
WriteTimeout: 10 * time.Second,
|
|
}
|
|
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
|
log.Printf("API server error: %v", err)
|
|
}
|
|
}
|
|
|
|
func verifySQLiteDB(dbPath, blacklistPath string, concurrency, limit, batchSize int, daemon bool, serverPort int) {
|
|
fmt.Printf("=== Starting High-Performance Email Verification on SQLite DB ===\n")
|
|
fmt.Printf("Database: %s\n", dbPath)
|
|
|
|
db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=10000")
|
|
if err != nil {
|
|
log.Fatalf("Failed to open SQLite DB: %v", err)
|
|
}
|
|
defer db.Close()
|
|
db.SetMaxOpenConns(1)
|
|
|
|
// Ensure target verification table exists
|
|
_, err = db.Exec(`
|
|
CREATE TABLE IF NOT EXISTS email_verifications (
|
|
email TEXT PRIMARY KEY,
|
|
status TEXT,
|
|
reason TEXT,
|
|
domain TEXT,
|
|
has_mx INTEGER,
|
|
is_disposable INTEGER,
|
|
is_role INTEGER,
|
|
is_free INTEGER,
|
|
suggestion TEXT,
|
|
verified_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_verif_status ON email_verifications(status);
|
|
|
|
CREATE VIEW IF NOT EXISTS v_clean_leads AS
|
|
SELECT l.*
|
|
FROM leads l
|
|
INNER JOIN email_verifications v ON l.email = v.email
|
|
WHERE v.status = 'clean';
|
|
|
|
CREATE VIEW IF NOT EXISTS v_bad_leads AS
|
|
SELECT l.*, v.status as verification_status, v.reason as verification_reason
|
|
FROM leads l
|
|
INNER JOIN email_verifications v ON l.email = v.email
|
|
WHERE v.status != 'clean';
|
|
`)
|
|
if err != nil {
|
|
log.Fatalf("Failed to create verification table/views: %v", err)
|
|
}
|
|
|
|
bv, err := NewBatchVerifier(blacklistPath, concurrency)
|
|
if err != nil {
|
|
log.Fatalf("Failed to initialize batch verifier: %v", err)
|
|
}
|
|
|
|
if daemon && serverPort > 0 {
|
|
go startAPIServer(bv, serverPort)
|
|
}
|
|
|
|
// Count already verified
|
|
var verifiedCount int
|
|
_ = db.QueryRow("SELECT COUNT(*) FROM email_verifications;").Scan(&verifiedCount)
|
|
var totalLeads int
|
|
_ = db.QueryRow("SELECT COUNT(*) FROM leads;").Scan(&totalLeads)
|
|
fmt.Printf("Database status: %d / %d leads verified so far.\n", verifiedCount, totalLeads)
|
|
|
|
chunkSize := 25000
|
|
totalToProcess := limit
|
|
if totalToProcess <= 0 {
|
|
totalToProcess = totalLeads - verifiedCount
|
|
}
|
|
fmt.Printf("Target to verify: %d emails with %d concurrent workers (chunk size: %d)...\n", totalToProcess, concurrency, chunkSize)
|
|
|
|
totalProcessed := 0
|
|
for {
|
|
curLimit := chunkSize
|
|
if limit > 0 {
|
|
remaining := limit - totalProcessed
|
|
if remaining <= 0 {
|
|
break
|
|
}
|
|
if remaining < curLimit {
|
|
curLimit = remaining
|
|
}
|
|
}
|
|
|
|
rows, err := db.Query("SELECT l.email FROM leads l LEFT JOIN email_verifications v ON l.email = v.email WHERE v.email IS NULL LIMIT ?", curLimit)
|
|
if err != nil {
|
|
log.Fatalf("Query error: %v", err)
|
|
}
|
|
|
|
var emailList []string
|
|
for rows.Next() {
|
|
var em string
|
|
if err := rows.Scan(&em); err == nil {
|
|
emailList = append(emailList, em)
|
|
}
|
|
}
|
|
rows.Close()
|
|
|
|
if len(emailList) == 0 {
|
|
if daemon {
|
|
fmt.Printf("[%s] No unverified leads in database. Sleeping 10s for incoming harvester leads...\n", time.Now().Format("15:04:05"))
|
|
time.Sleep(10 * time.Second)
|
|
continue
|
|
}
|
|
fmt.Println("All leads in database are verified!")
|
|
break
|
|
}
|
|
|
|
// Channels for work distribution
|
|
jobs := make(chan string, concurrency*4)
|
|
results := make(chan VerifyResult, concurrency*4)
|
|
|
|
var wg sync.WaitGroup
|
|
for w := 0; w < concurrency; w++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for em := range jobs {
|
|
results <- bv.VerifyOne(em)
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Inserter routine
|
|
doneInserter := make(chan struct{})
|
|
var chunkClean, chunkBounce, chunkNoMX, chunkDisp, chunkRole, chunkSyntax int64
|
|
|
|
go func() {
|
|
defer close(doneInserter)
|
|
batch := make([]VerifyResult, 0, batchSize)
|
|
|
|
flush := func() {
|
|
if len(batch) == 0 {
|
|
return
|
|
}
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
log.Printf("Tx begin error: %v", err)
|
|
return
|
|
}
|
|
stmt, err := tx.Prepare(`
|
|
INSERT OR REPLACE INTO email_verifications
|
|
(email, status, reason, domain, has_mx, is_disposable, is_role, is_free, suggestion)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`)
|
|
if err != nil {
|
|
_ = tx.Rollback()
|
|
log.Printf("Stmt prepare error: %v", err)
|
|
return
|
|
}
|
|
for _, r := range batch {
|
|
hasMxInt := 0
|
|
if r.HasMX {
|
|
hasMxInt = 1
|
|
}
|
|
dispInt := 0
|
|
if r.Disposable {
|
|
dispInt = 1
|
|
}
|
|
roleInt := 0
|
|
if r.Role {
|
|
roleInt = 1
|
|
}
|
|
freeInt := 0
|
|
if r.Free {
|
|
freeInt = 1
|
|
}
|
|
_, _ = stmt.Exec(r.Email, string(r.Status), r.Reason, r.Domain, hasMxInt, dispInt, roleInt, freeInt, r.Suggestion)
|
|
}
|
|
_ = stmt.Close()
|
|
if err := tx.Commit(); err != nil {
|
|
log.Printf("Tx commit error: %v", err)
|
|
}
|
|
batch = batch[:0]
|
|
}
|
|
|
|
for res := range results {
|
|
batch = append(batch, res)
|
|
switch res.Status {
|
|
case StatusClean:
|
|
atomic.AddInt64(&chunkClean, 1)
|
|
case StatusBounced:
|
|
atomic.AddInt64(&chunkBounce, 1)
|
|
case StatusBadNoMX:
|
|
atomic.AddInt64(&chunkNoMX, 1)
|
|
case StatusDisposable:
|
|
atomic.AddInt64(&chunkDisp, 1)
|
|
case StatusRoleAccount:
|
|
atomic.AddInt64(&chunkRole, 1)
|
|
case StatusBadSyntax:
|
|
atomic.AddInt64(&chunkSyntax, 1)
|
|
}
|
|
if len(batch) >= batchSize {
|
|
flush()
|
|
}
|
|
}
|
|
flush()
|
|
}()
|
|
|
|
t0 := time.Now()
|
|
for _, em := range emailList {
|
|
jobs <- em
|
|
}
|
|
close(jobs)
|
|
wg.Wait()
|
|
close(results)
|
|
<-doneInserter
|
|
|
|
totalProcessed += len(emailList)
|
|
elapsed := time.Since(t0).Seconds()
|
|
rate := float64(len(emailList)) / elapsed
|
|
fmt.Printf(" Chunk complete: %d verified in %.1fs (%.0f emails/sec) | Clean: %d, Bounces: %d, Bad MX: %d, Disp: %d\n",
|
|
len(emailList), elapsed, rate, chunkClean, chunkBounce, chunkNoMX, chunkDisp)
|
|
|
|
if limit > 0 && totalProcessed >= limit {
|
|
break
|
|
}
|
|
}
|
|
|
|
fmt.Printf("\n=== Verification Batch Complete ===\n")
|
|
printStats(db)
|
|
}
|
|
|
|
func verifyCSV(csvIn, csvOut, cleanOut, badOut, blacklistPath, dbPath string, concurrency int, recordDB bool) {
|
|
fmt.Printf("=== Verifying CSV: %s ===\n", csvIn)
|
|
f, err := os.Open(csvIn)
|
|
if err != nil {
|
|
log.Fatalf("Failed to open CSV: %v", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
reader := csv.NewReader(f)
|
|
header, err := reader.Read()
|
|
if err != nil {
|
|
log.Fatalf("Failed to read CSV header: %v", err)
|
|
}
|
|
|
|
emailIdx := 1
|
|
for i, h := range header {
|
|
if strings.Contains(strings.ToLower(h), "email") {
|
|
emailIdx = i
|
|
break
|
|
}
|
|
}
|
|
|
|
bv, err := NewBatchVerifier(blacklistPath, concurrency)
|
|
if err != nil {
|
|
log.Fatalf("Failed to create verifier: %v", err)
|
|
}
|
|
|
|
var rows [][]string
|
|
for {
|
|
r, err := reader.Read()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
continue
|
|
}
|
|
rows = append(rows, r)
|
|
}
|
|
total := len(rows)
|
|
fmt.Printf("Loaded %d rows from %s (email column index: %d)\n", total, csvIn, emailIdx)
|
|
fmt.Printf("Verifying with %d parallel workers...\n", concurrency)
|
|
|
|
type csvJob struct {
|
|
idx int
|
|
row []string
|
|
em string
|
|
}
|
|
type csvResult struct {
|
|
idx int
|
|
res VerifyResult
|
|
}
|
|
|
|
jobs := make(chan csvJob, concurrency*4)
|
|
results := make(chan csvResult, concurrency*4)
|
|
verifiedResults := make([]VerifyResult, total)
|
|
|
|
var wg sync.WaitGroup
|
|
for w := 0; w < concurrency; w++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for j := range jobs {
|
|
results <- csvResult{
|
|
idx: j.idx,
|
|
res: bv.VerifyOne(j.em),
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
doneCollector := make(chan struct{})
|
|
counts := make(map[VerificationStatus]int)
|
|
|
|
go func() {
|
|
defer close(doneCollector)
|
|
processed := 0
|
|
for cr := range results {
|
|
verifiedResults[cr.idx] = cr.res
|
|
counts[cr.res.Status]++
|
|
processed++
|
|
if processed%1000 == 0 || processed == total {
|
|
fmt.Printf(" Verified %d/%d (%.1f%%) | Clean: %d, Bounces: %d, Bad MX: %d, Disp: %d\n",
|
|
processed, total, float64(processed)/float64(total)*100,
|
|
counts[StatusClean], counts[StatusBounced], counts[StatusBadNoMX], counts[StatusDisposable])
|
|
}
|
|
}
|
|
}()
|
|
|
|
for i, r := range rows {
|
|
em := ""
|
|
if len(r) > emailIdx {
|
|
em = r[emailIdx]
|
|
}
|
|
jobs <- csvJob{idx: i, row: r, em: em}
|
|
}
|
|
close(jobs)
|
|
wg.Wait()
|
|
close(results)
|
|
<-doneCollector
|
|
|
|
// Annotated output CSV
|
|
if csvOut != "" {
|
|
outFile, err := os.Create(csvOut)
|
|
if err != nil {
|
|
log.Fatalf("Failed to create output CSV: %v", err)
|
|
}
|
|
defer outFile.Close()
|
|
|
|
writer := csv.NewWriter(outFile)
|
|
newHeader := append(header, "verification_status", "verification_reason", "has_mx", "is_disposable", "is_role")
|
|
_ = writer.Write(newHeader)
|
|
|
|
for i, r := range rows {
|
|
res := verifiedResults[i]
|
|
hasMXStr := "false"
|
|
if res.HasMX {
|
|
hasMXStr = "true"
|
|
}
|
|
dispStr := "false"
|
|
if res.Disposable {
|
|
dispStr = "true"
|
|
}
|
|
roleStr := "false"
|
|
if res.Role {
|
|
roleStr = "true"
|
|
}
|
|
outRow := append(r, string(res.Status), res.Reason, hasMXStr, dispStr, roleStr)
|
|
_ = writer.Write(outRow)
|
|
}
|
|
writer.Flush()
|
|
fmt.Printf("✅ Wrote %d verified results to %s\n", total, csvOut)
|
|
}
|
|
|
|
// Clean-only CSV
|
|
if cleanOut != "" {
|
|
cleanFile, err := os.Create(cleanOut)
|
|
if err != nil {
|
|
log.Fatalf("Failed to create clean CSV: %v", err)
|
|
}
|
|
defer cleanFile.Close()
|
|
cleanWriter := csv.NewWriter(cleanFile)
|
|
_ = cleanWriter.Write(header)
|
|
cleanCount := 0
|
|
for i, r := range rows {
|
|
if verifiedResults[i].Status == StatusClean {
|
|
_ = cleanWriter.Write(r)
|
|
cleanCount++
|
|
}
|
|
}
|
|
cleanWriter.Flush()
|
|
fmt.Printf("✅ Wrote %d 100%% clean deliverable rows to %s\n", cleanCount, cleanOut)
|
|
}
|
|
|
|
// Bad-only CSV (archive)
|
|
if badOut != "" {
|
|
badFile, err := os.Create(badOut)
|
|
if err != nil {
|
|
log.Fatalf("Failed to create bad CSV: %v", err)
|
|
}
|
|
defer badFile.Close()
|
|
badWriter := csv.NewWriter(badFile)
|
|
badHeader := append(header, "verification_status", "verification_reason")
|
|
_ = badWriter.Write(badHeader)
|
|
badCount := 0
|
|
for i, r := range rows {
|
|
if verifiedResults[i].Status != StatusClean {
|
|
outRow := append(r, string(verifiedResults[i].Status), verifiedResults[i].Reason)
|
|
_ = badWriter.Write(outRow)
|
|
badCount++
|
|
}
|
|
}
|
|
badWriter.Flush()
|
|
fmt.Printf("⚠️ Wrote %d bad/bounced rows to archive: %s\n", badCount, badOut)
|
|
}
|
|
|
|
// Record into SQLite DB
|
|
if recordDB && dbPath != "" {
|
|
db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=10000")
|
|
if err != nil {
|
|
log.Printf("Warning: failed to open SQLite DB to record results: %v", err)
|
|
} else {
|
|
defer db.Close()
|
|
db.SetMaxOpenConns(1)
|
|
tx, err := db.Begin()
|
|
if err == nil {
|
|
stmt, err := tx.Prepare(`
|
|
INSERT OR REPLACE INTO email_verifications
|
|
(email, status, reason, domain, has_mx, is_disposable, is_role, is_free, suggestion)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`)
|
|
if err == nil {
|
|
for _, r := range verifiedResults {
|
|
if r.Email == "" {
|
|
continue
|
|
}
|
|
hasMxInt := 0
|
|
if r.HasMX {
|
|
hasMxInt = 1
|
|
}
|
|
dispInt := 0
|
|
if r.Disposable {
|
|
dispInt = 1
|
|
}
|
|
roleInt := 0
|
|
if r.Role {
|
|
roleInt = 1
|
|
}
|
|
freeInt := 0
|
|
if r.Free {
|
|
freeInt = 1
|
|
}
|
|
_, _ = stmt.Exec(r.Email, string(r.Status), r.Reason, r.Domain, hasMxInt, dispInt, roleInt, freeInt, r.Suggestion)
|
|
}
|
|
_ = stmt.Close()
|
|
_ = tx.Commit()
|
|
fmt.Printf("📦 Recorded %d verification results into %s\n", len(verifiedResults), dbPath)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fmt.Printf("\n--- Verification Summary for %s ---\n", csvIn)
|
|
for st, c := range counts {
|
|
fmt.Printf(" %-16s: %6d (%5.1f%%)\n", st, c, float64(c)/float64(total)*100)
|
|
}
|
|
}
|
|
|
|
func printStats(db *sql.DB) {
|
|
fmt.Printf("\n=== SQLite Database Verification Statistics ===\n")
|
|
var total int
|
|
_ = db.QueryRow("SELECT COUNT(*) FROM email_verifications").Scan(&total)
|
|
fmt.Printf("Total Verified in DB: %d\n", total)
|
|
|
|
rows, err := db.Query(`
|
|
SELECT status, count(*) FROM email_verifications GROUP BY status ORDER BY count(*) DESC;
|
|
`)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
for rows.Next() {
|
|
var st string
|
|
var cnt int
|
|
if err := rows.Scan(&st, &cnt); err == nil {
|
|
pct := 0.0
|
|
if total > 0 {
|
|
pct = float64(cnt) / float64(total) * 100
|
|
}
|
|
fmt.Printf(" %-16s: %8d (%5.1f%%)\n", st, cnt, pct)
|
|
}
|
|
}
|
|
}
|
|
|
|
func main() {
|
|
dbPath := flag.String("db", "/Users/a/Downloads/scale_1m_batches/leads_master.db", "SQLite database path")
|
|
blacklistPath := flag.String("blacklist", "/Users/a/Downloads/segmented_campaign_lists/complaints_blacklist.json", "Blacklist JSON or text file")
|
|
csvIn := flag.String("csv", "", "Path to CSV file to verify")
|
|
csvOut := flag.String("out", "", "Path to output verified CSV (annotated)")
|
|
cleanOut := flag.String("clean-out", "", "Path to output clean-only CSV (drops bad/bounces)")
|
|
badOut := flag.String("bad-out", "", "Path to output bad-only CSV (archive)")
|
|
recordDB := flag.Bool("record-db", false, "Record verified emails into SQLite DB")
|
|
concurrency := flag.Int("concurrency", 64, "Number of concurrent verification workers")
|
|
limit := flag.Int("limit", 0, "Max number of emails to verify from DB (0 for all)")
|
|
batchSize := flag.Int("batch", 2000, "SQLite insert batch transaction size")
|
|
statsOnly := flag.Bool("stats", false, "Print verification stats from DB and exit")
|
|
daemon := flag.Bool("daemon", false, "Run continuously in background, verifying new leads as they arrive")
|
|
singleEmail := flag.String("email", "", "Verify a single email and print JSON result")
|
|
serverMode := flag.Bool("server", false, "Run standalone HTTP microservice serving /v1/email/verify")
|
|
serverPort := flag.Int("port", 8085, "HTTP server port for /v1/email/verify (default 8085)")
|
|
flag.Parse()
|
|
|
|
// 0. Standalone Server mode
|
|
if *serverMode {
|
|
bv, err := NewBatchVerifier(*blacklistPath, *concurrency)
|
|
if err != nil {
|
|
log.Fatalf("Error: %v", err)
|
|
}
|
|
startAPIServer(bv, *serverPort)
|
|
return
|
|
}
|
|
|
|
// 1. Single email mode
|
|
if *singleEmail != "" {
|
|
bv, err := NewBatchVerifier(*blacklistPath, 1)
|
|
if err != nil {
|
|
log.Fatalf("Error: %v", err)
|
|
}
|
|
res := bv.VerifyOne(*singleEmail)
|
|
data, _ := json.MarshalIndent(res, "", " ")
|
|
fmt.Println(string(data))
|
|
return
|
|
}
|
|
|
|
// 2. Stats only mode
|
|
if *statsOnly {
|
|
db, err := sql.Open("sqlite3", *dbPath)
|
|
if err != nil {
|
|
log.Fatalf("Failed to open DB: %v", err)
|
|
}
|
|
defer db.Close()
|
|
printStats(db)
|
|
return
|
|
}
|
|
|
|
// 3. CSV Mode
|
|
if *csvIn != "" {
|
|
verifyCSV(*csvIn, *csvOut, *cleanOut, *badOut, *blacklistPath, *dbPath, *concurrency, *recordDB)
|
|
return
|
|
}
|
|
|
|
// 4. SQLite DB Mode
|
|
verifySQLiteDB(*dbPath, *blacklistPath, *concurrency, *limit, *batchSize, *daemon, *serverPort)
|
|
}
|