mirror of
https://github.com/hanzoskill/causal-inference.git
synced 2026-10-30 21:08:40 +00:00
193 lines
6.4 KiB
Python
193 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Backfill causal action log from message history (WhatsApp, Discord, Slack).
|
|
|
|
Usage:
|
|
# WhatsApp via wacli
|
|
wacli search --after 2024-01-01 --from me --format json > /tmp/wa_sent.json
|
|
python backfill_messages.py /tmp/wa_sent.json --platform whatsapp
|
|
|
|
# Or query directly
|
|
python backfill_messages.py --platform whatsapp --days 90
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timezone, timedelta
|
|
from pathlib import Path
|
|
from collections import defaultdict
|
|
|
|
DEFAULT_LOG_PATH = "memory/causal/action_log.jsonl"
|
|
|
|
|
|
def parse_time(time_str):
|
|
"""Parse various time formats to ISO."""
|
|
if not time_str:
|
|
return None
|
|
if isinstance(time_str, (int, float)):
|
|
return datetime.fromtimestamp(time_str, tz=timezone.utc).isoformat()
|
|
for fmt in ["%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"]:
|
|
try:
|
|
return datetime.strptime(time_str, fmt).isoformat()
|
|
except ValueError:
|
|
continue
|
|
return time_str
|
|
|
|
|
|
def get_time_bucket(iso_time):
|
|
"""Categorize time into morning/afternoon/evening."""
|
|
if not iso_time:
|
|
return "unknown"
|
|
try:
|
|
dt = datetime.fromisoformat(iso_time.replace("Z", "+00:00"))
|
|
hour = dt.hour
|
|
if 5 <= hour < 12:
|
|
return "morning"
|
|
elif 12 <= hour < 17:
|
|
return "afternoon"
|
|
else:
|
|
return "evening"
|
|
except:
|
|
return "unknown"
|
|
|
|
|
|
def find_response(msg, all_messages, max_hours=48):
|
|
"""Find if there was a response to this message."""
|
|
chat_id = msg.get("chat_id") or msg.get("chatId") or msg.get("channel_id")
|
|
sent_time = parse_time(msg.get("timestamp") or msg.get("date") or msg.get("time"))
|
|
|
|
if not chat_id or not sent_time:
|
|
return False, None, None
|
|
|
|
sent_dt = datetime.fromisoformat(sent_time.replace("Z", "+00:00"))
|
|
max_response_time = sent_dt + timedelta(hours=max_hours)
|
|
|
|
for other in all_messages:
|
|
other_chat = other.get("chat_id") or other.get("chatId") or other.get("channel_id")
|
|
if other_chat != chat_id:
|
|
continue
|
|
|
|
# Skip if from me
|
|
if other.get("from_me") or other.get("fromMe") or other.get("is_mine"):
|
|
continue
|
|
|
|
other_time = parse_time(other.get("timestamp") or other.get("date") or other.get("time"))
|
|
if not other_time:
|
|
continue
|
|
|
|
other_dt = datetime.fromisoformat(other_time.replace("Z", "+00:00"))
|
|
|
|
# Check if it's a response (after sent, within window)
|
|
if sent_dt < other_dt <= max_response_time:
|
|
delay_hours = (other_dt - sent_dt).total_seconds() / 3600
|
|
return True, other_time, delay_hours
|
|
|
|
return False, None, None
|
|
|
|
|
|
def backfill_from_json(json_path, log_path, platform, all_messages=None):
|
|
"""Backfill action log from JSON export."""
|
|
with open(json_path) as f:
|
|
messages = json.load(f)
|
|
|
|
if isinstance(messages, dict):
|
|
messages = messages.get("messages", [])
|
|
|
|
if all_messages is None:
|
|
all_messages = messages
|
|
|
|
# Filter to only sent messages
|
|
sent_messages = [
|
|
m for m in messages
|
|
if m.get("from_me") or m.get("fromMe") or m.get("is_mine")
|
|
]
|
|
|
|
path = Path(log_path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
count = 0
|
|
for msg in sent_messages:
|
|
sent_time = parse_time(msg.get("timestamp") or msg.get("date") or msg.get("time"))
|
|
response_received, response_time, delay_hours = find_response(msg, all_messages)
|
|
|
|
message_text = msg.get("body") or msg.get("content") or msg.get("text") or ""
|
|
|
|
event = {
|
|
"action": "send_message",
|
|
"domain": "messaging",
|
|
"context": {
|
|
"platform": platform,
|
|
"message_length": len(message_text),
|
|
"has_media": bool(msg.get("hasMedia") or msg.get("media")),
|
|
"is_group": bool(msg.get("isGroup") or msg.get("is_group")),
|
|
},
|
|
"time": sent_time,
|
|
"pre_state": {
|
|
"time_bucket": get_time_bucket(sent_time),
|
|
},
|
|
"post_state": {
|
|
"response_received": response_received,
|
|
"response_delay_hours": delay_hours,
|
|
},
|
|
"outcome": "response" if response_received else "no_response",
|
|
"outcome_observed_at": response_time or datetime.now(timezone.utc).isoformat(),
|
|
"backfilled": True,
|
|
}
|
|
|
|
with open(path, "a") as f:
|
|
f.write(json.dumps(event) + "\n")
|
|
count += 1
|
|
|
|
return count
|
|
|
|
|
|
def backfill_whatsapp(days, log_path):
|
|
"""Backfill from WhatsApp via wacli."""
|
|
after_date = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
|
|
|
|
result = subprocess.run(
|
|
["wacli", "search", "--after", after_date, "--limit", "1000", "--format", "json"],
|
|
capture_output=True, text=True
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
print(f"Error running wacli: {result.stderr}", file=sys.stderr)
|
|
return 0
|
|
|
|
all_messages = json.loads(result.stdout) if result.stdout.strip() else []
|
|
|
|
tmp_path = "/tmp/wacli_messages.json"
|
|
with open(tmp_path, "w") as f:
|
|
json.dump(all_messages, f)
|
|
|
|
return backfill_from_json(tmp_path, log_path, "whatsapp", all_messages)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Backfill message actions")
|
|
parser.add_argument("json_file", nargs="?", help="JSON file with message data")
|
|
parser.add_argument("--platform", "-p", default="whatsapp",
|
|
choices=["whatsapp", "discord", "slack", "telegram"],
|
|
help="Messaging platform")
|
|
parser.add_argument("--days", "-d", type=int, default=90, help="Days to look back")
|
|
parser.add_argument("--log-path", "-l", default=DEFAULT_LOG_PATH, help="Action log path")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.json_file:
|
|
count = backfill_from_json(args.json_file, args.log_path, args.platform)
|
|
elif args.platform == "whatsapp":
|
|
count = backfill_whatsapp(args.days, args.log_path)
|
|
else:
|
|
print(f"Direct query not implemented for {args.platform}. Provide JSON file.",
|
|
file=sys.stderr)
|
|
return
|
|
|
|
print(f"Backfilled {count} message actions to {args.log_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|