initial setup

This commit is contained in:
DJ Gillespie 2025-09-23 19:01:30 -06:00
commit 15aa874800
51 changed files with 1605 additions and 0 deletions

89
pythonechoserver.py Normal file
View File

@ -0,0 +1,89 @@
#!/usr/bin/env python3
from http.server import HTTPServer, BaseHTTPRequestHandler
import json
from datetime import datetime
from urllib.parse import urlparse, parse_qs
class WebhookHandler(BaseHTTPRequestHandler):
def log_message(self, format, *args):
# Override to add timestamp
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{timestamp}] {format % args}")
def do_POST(self):
self.handle_request()
def do_GET(self):
self.handle_request()
def do_PUT(self):
self.handle_request()
def do_DELETE(self):
self.handle_request()
def handle_request(self):
print("=" * 80)
print(f"📧 WEBHOOK RECEIVED - {self.command} {self.path}")
print("=" * 80)
# Log basic info
print(f"Method: {self.command}")
print(f"Path: {self.path}")
print(f"Remote IP: {self.client_address[0]}")
# Log headers
print("\n📋 HEADERS:")
for header, value in self.headers.items():
print(f" {header}: {value}")
# Read and log body
content_length = int(self.headers.get('Content-Length', 0))
if content_length > 0:
body = self.rfile.read(content_length)
print(f"\n📄 RAW BODY:")
print(f" {body.decode('utf-8', errors='replace')}")
# Try to parse JSON
try:
if self.headers.get('Content-Type', '').startswith('application/json'):
json_data = json.loads(body)
print(f"\n🎯 JSON PAYLOAD:")
print(json.dumps(json_data, indent=2))
# Highlight specific fields
if isinstance(json_data, dict):
for key in ['email', 'event', 'subject', 'sender', 'timestamp']:
if key in json_data:
print(f" {key.upper()}: {json_data[key]}")
except:
pass
print("=" * 80)
print()
# Send response
self.send_response(200)
self.send_header('Content-Type', 'application/json')
self.end_headers()
response = {
"status": "received",
"method": self.command,
"path": self.path,
"timestamp": datetime.now().isoformat()
}
self.wfile.write(json.dumps(response).encode())
if __name__ == '__main__':
print("🚀 Starting Simple Webhook Echo Server...")
print("📡 Listening on http://localhost:8080")
print("⚡ Press Ctrl+C to stop")
print("=" * 80)
server = HTTPServer(('0.0.0.0', 8080), WebhookHandler)
try:
server.serve_forever()
except KeyboardInterrupt:
print("\n\n👋 Shutting down server...")
server.server_close()

11
requirements.txt Normal file
View File

@ -0,0 +1,11 @@
Django==6.0a1
djangorestframework==3.16.1
django-cors-headers==4.9.0
psycopg2-binary==2.9.10
cryptography==46.0.1
requests==2.32.5
imapclient==3.0.1
celery==5.5.3
redis==6.4.0
python-dotenv==1.1.1
gunicorn==23.0.0

0
src/core/__init__.py Normal file
View File

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

16
src/core/asgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
ASGI config for imap_relay project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.1/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
application = get_asgi_application()

BIN
src/core/db.sqlite3 Normal file

Binary file not shown.

View File

@ -0,0 +1,175 @@
"""
Django settings for imap_relay project.
Generated by 'django-admin startproject' using Django 3.1.
For more information on this file, see
https://docs.djangoproject.com/en/3.1/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.1/ref/settings/
"""
import os
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'd$kxa%nt#t1td&$$2%vg+ec&%!6fn*(ii)@kx)wb7sm183^bim'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = os.getenv('DEBUG', 'True').lower() == 'true'
ALLOWED_HOSTS = ['*']
CORS_ALLOW_ALL_ORIGINS = True
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'rest_framework',
'corsheaders',
'relay',
]
MIDDLEWARE = [
'corsheaders.middleware.CorsMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'core.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'core.wsgi.application'
ASGI_APPLICATION = 'core.asgi.application'
# Database
# https://docs.djangoproject.com/en/3.1/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'db.sqlite3',
}
}
# REST Framework
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': [
'relay.utils.authentication.APITokenAuthentication',
],
'DEFAULT_PERMISSION_CLASSES': [
'rest_framework.permissions.AllowAny', # Allow any by default
],
'DEFAULT_RENDERER_CLASSES': [
'rest_framework.renderers.JSONRenderer',
],
}
# Password validation
# https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Logging
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '{levelname} {asctime} {module} {message}',
'style': '{',
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'verbose',
},
},
'root': {
'handlers': ['console'],
'level': 'INFO',
},
'loggers': {
'relay.utils.imap_manager': {
'handlers': ['console'],
'level': 'INFO',
'propagate': False,
},
},
}
# Internationalization
# https://docs.djangoproject.com/en/3.1/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'MST'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.1/howto/static-files/
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
API_TOKEN = os.getenv('API_TOKEN', 'v3rys3cr37k3y')
ENCRYPTION_KEY = os.getenv('ENCRYPTION_KEY', 'cKJj0iMVeg9gwGjN_D6aCYayg-gCBe-uO8mPp6rwz-8=')

Binary file not shown.

Binary file not shown.

22
src/core/urls.py Normal file
View File

@ -0,0 +1,22 @@
"""imap_relay URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
urlpatterns = [
path('admin/', admin.site.urls),
path('api/', include('relay.urls')),
]

16
src/core/wsgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
WSGI config for imap_relay project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.1/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
application = get_wsgi_application()

22
src/manage.py Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

25
src/relay/admin.py Normal file
View File

@ -0,0 +1,25 @@
from django.contrib import admin
from relay.models import IMAPAccount
@admin.register(IMAPAccount)
class IMAPAccountAdmin(admin.ModelAdmin):
list_display = ['email', 'imap_server', 'auth_type', 'is_active', 'last_activity', 'created_at']
list_filter = ['auth_type', 'is_active', 'imap_server']
search_fields = ['email', 'username']
readonly_fields = ['created_at', 'updated_at', 'last_activity']
fieldsets = (
(None, {
'fields': ('email', 'username', 'webhook_url')
}),
('IMAP Settings', {
'fields': ('imap_server', 'imap_port', 'auth_type')
}),
('Status', {
'fields': ('is_active', 'last_activity')
}),
('Timestamps', {
'fields': ('created_at', 'updated_at'),
'classes': ('collapse',)
})
)

23
src/relay/apps.py Normal file
View File

@ -0,0 +1,23 @@
from django.apps import AppConfig
import logging
logger = logging.getLogger(__name__)
class RelayConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'relay'
def ready(self):
# Start IMAP connection manager when Django starts
from relay.utils.imap_manager import connection_manager
import threading
# Start in a separate thread to avoid blocking Django startup
def start_manager():
try:
connection_manager.start_manager()
except Exception as e:
logger.error(f"Failed to start IMAP manager: {e}")
thread = threading.Thread(target=start_manager, daemon=True)
thread.start()

View File

View File

@ -0,0 +1,43 @@
from django.core.management.base import BaseCommand
from relay.models import IMAPAccount
import imaplib
class Command(BaseCommand):
help = 'List all IMAP folders for an account'
def add_arguments(self, parser):
parser.add_argument('email', help='Email address to check folders for')
def handle(self, *args, **options):
email = options['email']
try:
account = IMAPAccount.objects.get(email=email)
except IMAPAccount.DoesNotExist:
self.stderr.write(f"Account {email} not found")
return
try:
imap = imaplib.IMAP4_SSL(account.imap_server, account.imap_port)
imap.login(account.username, account.password)
result, folder_list = imap.list()
if result == 'OK':
self.stdout.write(f"📁 Folders for {email}:")
for folder_line in folder_list:
folder_str = folder_line.decode('utf-8')
self.stdout.write(f" {folder_str}")
# Try to extract just the folder name
parts = folder_str.split(' "/" ')
if len(parts) >= 2:
folder_name = parts[1].strip('"')
if 'sent' in folder_name.lower():
self.stdout.write(f" ✅ SENT FOLDER: {folder_name}")
else:
self.stderr.write(f"Failed to list folders: {result}")
imap.logout()
except Exception as e:
self.stderr.write(f"Error: {e}")

View File

@ -0,0 +1,76 @@
from django.core.management.base import BaseCommand
from relay.utils.imap_manager import connection_manager
from relay.models import IMAPAccount
import time
class Command(BaseCommand):
help = 'Start IMAP IDLE connections for all active accounts'
def add_arguments(self, parser):
parser.add_argument(
'--restart',
action='store_true',
help='Stop existing connections and restart them',
)
def handle(self, *args, **options):
if options['restart']:
self.stdout.write("🛑 Stopping existing connections...")
connection_manager.stop_manager()
time.sleep(1)
self.stdout.write("🚀 Starting IMAP Connection Manager...")
# Always start the manager fresh
if not connection_manager.running:
connection_manager.start_manager()
# Give it a moment to start
time.sleep(2)
# Force load accounts (ignore the table check)
try:
active_accounts = IMAPAccount.objects.filter(is_active=True)
self.stdout.write(f"📧 Found {active_accounts.count()} active accounts")
if active_accounts.count() == 0:
self.stdout.write(self.style.WARNING("⚠️ No active accounts found"))
return
for account in active_accounts:
if account.email not in connection_manager.connections:
self.stdout.write(f" Adding connections for {account.email}")
try:
connection_manager.add_account(account)
self.stdout.write(f"✅ Successfully added {account.email}")
except Exception as e:
self.stdout.write(
self.style.ERROR(f"❌ Failed to add {account.email}: {e}")
)
else:
self.stdout.write(f"{account.email} already connected")
# Show detailed status
time.sleep(2) # Give connections time to establish
total_connections = sum(len(conns) for conns in connection_manager.connections.values())
self.stdout.write(
self.style.SUCCESS(
f"✅ IMAP Manager running with {total_connections} total connections "
f"across {len(connection_manager.connections)} accounts"
)
)
# List all connections with their status
for email, connections in connection_manager.connections.items():
folders = []
for conn in connections:
status = "🟢" if conn.running else "🔴"
folders.append(f"{status}{conn.folder_name}")
self.stdout.write(f" 📁 {email}: {', '.join(folders)}")
except Exception as e:
self.stdout.write(
self.style.ERROR(f"❌ Error loading accounts: {e}")
)
import traceback
traceback.print_exc()

View File

@ -0,0 +1,36 @@
# Generated by Django 6.0a1 on 2025-09-23 23:22
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='IMAPAccount',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('email', models.EmailField(db_index=True, max_length=254, unique=True)),
('imap_server', models.CharField(max_length=255)),
('imap_port', models.IntegerField(default=993)),
('username', models.CharField(max_length=255)),
('_password', models.BinaryField(blank=True, null=True)),
('_oauth_token', models.BinaryField(blank=True, null=True)),
('auth_type', models.CharField(choices=[('password', 'Password'), ('oauth', 'OAuth')], default='password', max_length=50)),
('webhook_url', models.URLField()),
('is_active', models.BooleanField(default=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('last_activity', models.DateTimeField(blank=True, null=True)),
],
options={
'db_table': 'imap_accounts',
'ordering': ['-created_at'],
},
),
]

View File

63
src/relay/models.py Normal file
View File

@ -0,0 +1,63 @@
from django.db import models
from django.utils import timezone
from relay.utils.encryption import encryption
class IMAPAccount(models.Model):
AUTH_TYPES = [
('password', 'Password'),
('oauth', 'OAuth'),
]
email = models.EmailField(unique=True, db_index=True)
imap_server = models.CharField(max_length=255)
imap_port = models.IntegerField(default=993)
username = models.CharField(max_length=255)
_password = models.BinaryField(blank=True, null=True)
_oauth_token = models.BinaryField(blank=True, null=True)
auth_type = models.CharField(max_length=50, choices=AUTH_TYPES, default='password')
webhook_url = models.URLField()
is_active = models.BooleanField(default=True)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
last_activity = models.DateTimeField(blank=True, null=True)
class Meta:
db_table = 'imap_accounts'
ordering = ['-created_at']
def __str__(self):
return self.email
@property
def password(self):
if self._password:
return encryption.decrypt(self._password)
return None
@password.setter
def password(self, pwd):
if pwd:
self._password = encryption.encrypt(pwd)
else:
self._password = None
@property
def oauth_token(self):
if self._oauth_token:
return encryption.decrypt(self._oauth_token)
return None
@oauth_token.setter
def oauth_token(self, token):
if token:
self._oauth_token = encryption.encrypt(token)
else:
self._oauth_token = None
def update_activity(self):
self.last_activity = timezone.now()
self.save(update_fields=['last_activity'])
def set_inactive(self):
self.is_active = False
self.save(update_fields=['is_active'])

68
src/relay/serializers.py Normal file
View File

@ -0,0 +1,68 @@
from rest_framework import serializers
from relay.models import IMAPAccount
class IMAPAccountCreateSerializer(serializers.ModelSerializer):
password = serializers.CharField(write_only=True, required=False, allow_blank=True)
oauth_token = serializers.CharField(write_only=True, required=False, allow_blank=True)
class Meta:
model = IMAPAccount
fields = [
'email', 'imap_server', 'imap_port', 'username',
'password', 'oauth_token', 'auth_type', 'webhook_url'
]
def create(self, validated_data):
password = validated_data.pop('password', None)
oauth_token = validated_data.pop('oauth_token', None)
account = IMAPAccount.objects.create(**validated_data)
if password:
account.password = password
if oauth_token:
account.oauth_token = oauth_token
account.save()
return account
class IMAPAccountUpdateSerializer(serializers.ModelSerializer):
password = serializers.CharField(write_only=True, required=False, allow_blank=True)
oauth_token = serializers.CharField(write_only=True, required=False, allow_blank=True)
class Meta:
model = IMAPAccount
fields = ['webhook_url', 'password', 'oauth_token', 'is_active']
def update(self, instance, validated_data):
password = validated_data.pop('password', None)
oauth_token = validated_data.pop('oauth_token', None)
for attr, value in validated_data.items():
setattr(instance, attr, value)
if password:
instance.password = password
if oauth_token:
instance.oauth_token = oauth_token
instance.save()
return instance
class IMAPAccountSerializer(serializers.ModelSerializer):
class Meta:
model = IMAPAccount
fields = [
'id', 'email', 'imap_server', 'imap_port', 'username',
'auth_type', 'webhook_url', 'is_active', 'created_at',
'updated_at', 'last_activity'
]
read_only_fields = ['id', 'created_at', 'updated_at']
class WebhookPayloadSerializer(serializers.Serializer):
email = serializers.EmailField()
event = serializers.CharField()
message_id = serializers.CharField(required=False, allow_blank=True)
subject = serializers.CharField(required=False, allow_blank=True)
sender = serializers.CharField(required=False, allow_blank=True)
timestamp = serializers.DateTimeField()

10
src/relay/urls.py Normal file
View File

@ -0,0 +1,10 @@
from django.urls import path, include
from rest_framework.routers import DefaultRouter
from relay.views import IMAPAccountViewSet
router = DefaultRouter()
router.register(r'accounts', IMAPAccountViewSet, basename='imap-accounts')
urlpatterns = [
path('', include(router.urls)),
]

Binary file not shown.

View File

@ -0,0 +1,48 @@
from rest_framework import authentication
from rest_framework import exceptions
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
import logging
logger = logging.getLogger(__name__)
class APITokenUser:
"""A simple user-like object for API token authentication."""
def __init__(self, token):
self.token = token
self.is_authenticated = True
self.is_anonymous = False
self.is_active = True
def __str__(self):
return f"APITokenUser({self.token[:10]}...)"
class APITokenAuthentication(authentication.BaseAuthentication):
def authenticate(self, request):
auth_header = request.META.get('HTTP_AUTHORIZATION')
logger.info(f"Auth header received: '{auth_header}'")
logger.info(f"Expected token: '{settings.API_TOKEN}'")
if not auth_header:
logger.info("No Authorization header found")
return None
if not auth_header.startswith('Bearer '):
logger.error(f"Invalid authorization format: '{auth_header}'")
raise exceptions.AuthenticationFailed('Invalid authorization format')
token = auth_header.replace('Bearer ', '')
logger.info(f"Extracted token: '{token}'")
if not settings.API_TOKEN:
logger.error("API_TOKEN setting is not configured")
raise exceptions.AuthenticationFailed('Server configuration error')
if token != settings.API_TOKEN:
logger.error(f"Token mismatch. Received: '{token}', Expected: '{settings.API_TOKEN}'")
raise exceptions.AuthenticationFailed('Invalid API token')
logger.info("Authentication successful")
# Return our custom API token user
return (APITokenUser(token), token)

View File

@ -0,0 +1,22 @@
import os
from cryptography.fernet import Fernet
from django.conf import settings
class CredentialEncryption:
def __init__(self):
key = settings.ENCRYPTION_KEY
if not key:
raise ValueError("ENCRYPTION_KEY setting not configured")
self.fernet = Fernet(key.encode())
def encrypt(self, data: str) -> bytes:
if not data:
return b''
return self.fernet.encrypt(data.encode())
def decrypt(self, encrypted_data: bytes) -> str:
if not encrypted_data:
return ''
return self.fernet.decrypt(encrypted_data).decode()
encryption = CredentialEncryption()

View File

@ -0,0 +1,713 @@
import threading
import time
import logging
import imaplib
import email
import requests
import socket
import select
from datetime import datetime
from django.utils import timezone
from django.db import connection
logger = logging.getLogger(__name__)
class IMAPConnectionManager:
def __init__(self):
self.connections = {} # email -> list of IMAPConnection instances
self.running = False
self.manager_thread = None
def start_manager(self):
if not self.running:
self.running = True
self.manager_thread = threading.Thread(target=self._run_manager, daemon=True)
self.manager_thread.start()
logger.info("IMAP Connection Manager started")
def stop_manager(self):
self.running = False
for email_connections in list(self.connections.values()):
for connection in email_connections:
connection.stop()
self.connections.clear()
if self.manager_thread:
self.manager_thread.join(timeout=5)
logger.info("IMAP Connection Manager stopped")
def _run_manager(self):
self._load_accounts()
while self.running:
try:
self._health_check()
time.sleep(60)
except Exception as e:
logger.error(f"Manager loop error: {e}")
def add_account(self, account):
if account.email not in self.connections:
self.connections[account.email] = []
# Get folders to monitor for this account
folders = self._get_folders_to_monitor(account)
for folder in folders:
connection = IMAPConnection(account, folder)
self.connections[account.email].append(connection)
connection.start()
logger.info(f"Added IMAP connections for {account.email} monitoring {len(folders)} folders: {folders}")
def remove_account(self, email):
if email in self.connections:
for connection in self.connections[email]:
connection.stop()
del self.connections[email]
logger.info(f"Removed IMAP connections for {email}")
def update_account(self, email, updated_account):
if email in self.connections:
# Stop old connections
for connection in self.connections[email]:
connection.stop()
del self.connections[email]
# Start new connections
self.add_account(updated_account)
def _get_folders_to_monitor(self, account):
"""Get list of folders to monitor based on email provider."""
# Gmail
if 'gmail.com' in account.imap_server.lower():
return ['INBOX', '[Gmail]/Sent Mail']
# Outlook/Hotmail/Live
elif any(provider in account.imap_server.lower() for provider in ['outlook', 'hotmail', 'live']):
return ['INBOX', 'Sent Items']
# Yahoo
elif 'yahoo' in account.imap_server.lower():
return ['INBOX', 'Sent']
# iCloud
elif 'icloud' in account.imap_server.lower():
return ['INBOX', 'Sent Messages']
# Generic/Other providers - try common folder names
else:
return ['INBOX', 'Sent']
def _load_accounts(self):
try:
if not self._table_exists('imap_accounts'):
logger.info("imap_accounts table does not exist yet, skipping account loading")
return
from relay.models import IMAPAccount
accounts = IMAPAccount.objects.filter(is_active=True)
for account in accounts:
self.add_account(account)
logger.info(f"Loaded {len(accounts)} IMAP accounts")
except Exception as e:
logger.error(f"Error loading accounts: {e}")
def _table_exists(self, table_name):
try:
with connection.cursor() as cursor:
cursor.execute("""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_name = %s
);
""", [table_name])
return cursor.fetchone()[0]
except Exception:
return False
def _health_check(self):
if not self.connections:
return
# Check all connections for all accounts
accounts_to_restart = []
for email, email_connections in list(self.connections.items()):
unhealthy_connections = []
for connection in email_connections:
if not connection.is_healthy():
unhealthy_connections.append(connection)
if unhealthy_connections:
accounts_to_restart.append(email)
for email in accounts_to_restart:
try:
logger.warning(f"Restarting unhealthy connections for {email}")
from relay.models import IMAPAccount
account = IMAPAccount.objects.filter(email=email, is_active=True).first()
if account:
self.remove_account(email)
self.add_account(account)
else:
self.remove_account(email)
except Exception as e:
logger.error(f"Error during health check for {email}: {e}")
class IMAPConnection:
def __init__(self, account, folder_name):
self.account = account
self.folder_name = folder_name
self.imap = None
self.running = False
self.last_activity = datetime.now()
self.reconnect_attempts = 0
self.max_reconnects = 5
self.connection_thread = None
self.last_uid = None
self._has_pending_messages = False
# Determine if this is an outgoing folder
self.is_outgoing_folder = self._is_outgoing_folder(folder_name)
def _is_outgoing_folder(self, folder_name):
"""Determine if this folder contains outgoing messages."""
outgoing_keywords = ['sent', 'outbox', 'draft']
return any(keyword in folder_name.lower() for keyword in outgoing_keywords)
def start(self):
if not self.running:
self.running = True
self.connection_thread = threading.Thread(target=self._connection_loop, daemon=True)
self.connection_thread.start()
def stop(self):
self.running = False
if self.imap:
try:
self.imap.send(b'DONE\r\n')
time.sleep(0.1)
self.imap.close()
self.imap.logout()
except:
pass
self.imap = None
if self.connection_thread:
self.connection_thread.join(timeout=5)
def is_healthy(self):
return (datetime.now() - self.last_activity).seconds < 2100
def _connection_loop(self):
while self.running and self.reconnect_attempts < self.max_reconnects:
try:
self._connect_and_idle()
except Exception as e:
logger.error(f"IMAP error for {self.account.email}[{self.folder_name}]: {e}")
self.reconnect_attempts += 1
if self.running:
wait_time = min(60 * self.reconnect_attempts, 300)
logger.info(f"Waiting {wait_time}s before reconnecting {self.account.email}[{self.folder_name}]")
time.sleep(wait_time)
if self.reconnect_attempts >= self.max_reconnects:
logger.error(f"Max reconnection attempts reached for {self.account.email}[{self.folder_name}]")
self._update_account_status(False)
def _connect_and_idle(self):
try:
# Create IMAP connection
self.imap = imaplib.IMAP4_SSL(self.account.imap_server, self.account.imap_port)
# Authenticate
if self.account.password:
self.imap.login(self.account.username, self.account.password)
else:
raise Exception("No password available for authentication")
# Robust folder selection with multiple strategies
folder_selected = self._select_folder_with_fallback()
if not folder_selected:
logger.error(f"❌ Could not select any folder for {self.account.email}[{self.folder_name}], aborting connection")
return
logger.info(f"✅ Connected to IMAP for {self.account.email}[{self.folder_name}]")
self.reconnect_attempts = 0
self._update_account_activity()
# Initialize last_uid if not set
if self.last_uid is None:
self._initialize_last_uid()
# IDLE loop
while self.running:
try:
# Reset pending messages flag
self._has_pending_messages = False
# Send IDLE command
tag = self.imap._new_tag()
command = f'{tag} IDLE\r\n'
self.imap.send(command.encode())
# Read the continuation response (+ idling)
response = self.imap.readline()
logger.debug(f"IDLE response for {self.account.email}[{self.folder_name}]: {response}")
# Check for correct IDLE continuation response
if b'+' not in response or b'idling' not in response.lower():
logger.error(f"Unexpected IDLE response for {self.account.email}[{self.folder_name}]: {response}")
break
logger.info(f"🔄 IDLE started successfully for {self.account.email}[{self.folder_name}]")
# Monitor for server notifications
idle_start = time.time()
while self.running and (time.time() - idle_start) < 1740: # 29 minutes
try:
# Check for incoming data with timeout
ready = select.select([self.imap.sock], [], [], 30)
if ready[0]:
# Data available - read it
try:
response = self.imap.readline()
response_str = response.decode('utf-8', errors='ignore').strip()
logger.debug(f"IDLE notification for {self.account.email}[{self.folder_name}]: {response_str}")
# Check for new message notification
if 'EXISTS' in response_str:
exists_count = self._parse_exists_response(response_str)
message_type = "outgoing" if self.is_outgoing_folder else "incoming"
logger.info(f"📧 New {message_type} message detected for {self.account.email}[{self.folder_name}] (total: {exists_count})")
# Handle new messages and break out of IDLE loop
self._handle_new_messages()
break
elif 'EXPUNGE' in response_str:
logger.info(f"🗑️ Message deleted for {self.account.email}[{self.folder_name}]")
elif 'FETCH' in response_str:
logger.info(f"🏷️ Message flags changed for {self.account.email}[{self.folder_name}]")
except socket.timeout:
continue
except Exception as e:
logger.debug(f"Error reading IDLE response for {self.account.email}[{self.folder_name}]: {e}")
continue
else:
# No data - continue monitoring
logger.debug(f"⏰ IDLE timeout - continuing for {self.account.email}[{self.folder_name}]")
except Exception as e:
logger.warning(f"Error during IDLE monitoring for {self.account.email}[{self.folder_name}]: {e}")
break
# End IDLE properly (if we haven't already)
if not self._has_pending_messages:
logger.info(f"⏹️ Ending IDLE naturally for {self.account.email}[{self.folder_name}]")
self.imap.send(b'DONE\r\n')
# Read any completion responses
try:
done_response = self.imap.readline()
logger.debug(f"IDLE completion response for {self.account.email}[{self.folder_name}]: {done_response}")
# Clear any remaining responses
response_count = 0
while response_count < 5: # Limit to prevent infinite loop
ready = select.select([self.imap.sock], [], [], 0.1)
if ready[0]:
response = self.imap.readline()
logger.debug(f"Additional IDLE response for {self.account.email}[{self.folder_name}]: {response}")
response_count += 1
else:
break
except Exception as e:
logger.debug(f"Error reading IDLE completion for {self.account.email}[{self.folder_name}]: {e}")
self.last_activity = datetime.now()
self._update_account_activity()
# Process any pending messages now
if self._has_pending_messages:
self._process_pending_messages()
# Brief pause before restarting IDLE
if self.running:
time.sleep(1)
except Exception as e:
logger.error(f"IDLE error for {self.account.email}[{self.folder_name}]: {e}")
break
except Exception as e:
logger.error(f"Connection error for {self.account.email}[{self.folder_name}]: {e}")
raise
def _select_folder_with_fallback(self):
"""
Robust folder selection with multiple strategies and fallbacks.
Returns True if a folder was successfully selected, False otherwise.
"""
logger.info(f"🔍 Attempting to select folder '{self.folder_name}' for {self.account.email}")
# Strategy 1: Try the folder name as-is
if self._try_select_folder(self.folder_name):
return True
# Strategy 2: For Gmail folders, try different quoting methods
if '[Gmail]' in self.folder_name:
gmail_strategies = [
f'"{self.folder_name}"', # Full quoted
self.folder_name.replace('[Gmail]/', ''), # Remove [Gmail]/ prefix
f'"[Gmail]"/"{self.folder_name.split("/")[1]}"', # Quote parts separately
self.folder_name.replace(' ', '\\ '), # Escape spaces
]
for strategy in gmail_strategies:
logger.debug(f"Trying Gmail strategy: {strategy}")
if self._try_select_folder(strategy):
self.folder_name = strategy # Update to working name
return True
# Strategy 3: Try with different quoting
quoting_strategies = [
f'"{self.folder_name}"', # Add quotes
self.folder_name.replace('"', ''), # Remove quotes
self.folder_name.replace(' ', '\\ '), # Escape spaces
]
for strategy in quoting_strategies:
if strategy != self.folder_name: # Don't repeat what we already tried
logger.debug(f"Trying quoting strategy: {strategy}")
if self._try_select_folder(strategy):
self.folder_name = strategy
return True
# Strategy 4: Try alternative folder names
logger.info(f"⚠️ Primary folder selection failed, trying alternatives for {self.account.email}")
alternative_folders = self._get_alternative_folder_names()
for alt_folder in alternative_folders[:5]: # Limit to first 5 alternatives
logger.debug(f"Trying alternative folder: {alt_folder}")
if self._try_select_folder(alt_folder):
logger.info(f"✅ Using alternative folder '{alt_folder}' instead of '{self.folder_name}' for {self.account.email}")
self.folder_name = alt_folder
return True
logger.error(f"❌ All folder selection strategies failed for {self.account.email}")
return False
def _try_select_folder(self, folder_name):
"""
Try to select a specific folder name.
Returns True if successful, False otherwise.
"""
try:
result, data = self.imap.select(folder_name)
if result == 'OK':
message_count = int(data[0]) if data and data[0] else 0
logger.info(f"✅ Successfully selected folder '{folder_name}' with {message_count} messages")
return True
else:
logger.debug(f"❌ Failed to select '{folder_name}': {result} - {data}")
return False
except Exception as e:
logger.debug(f"❌ Exception selecting '{folder_name}': {e}")
return False
def _get_alternative_folder_names(self):
"""Get alternative folder names to try if the primary folder doesn't exist."""
if self.is_outgoing_folder:
return ['Sent', 'Sent Items', 'Sent Messages', '[Gmail]/Sent Mail', 'INBOX.Sent']
else:
return ['INBOX']
def _initialize_last_uid(self):
"""Initialize last_uid to the latest message UID to avoid processing old messages."""
try:
result, messages = self.imap.uid('search', None, 'ALL')
if result == 'OK' and messages[0]:
uid_list = messages[0].split()
if uid_list:
self.last_uid = uid_list[-1].decode() if isinstance(uid_list[-1], bytes) else str(uid_list[-1])
logger.info(f"Initialized last_uid for {self.account.email}[{self.folder_name}]: {self.last_uid}")
else:
self.last_uid = "0"
else:
self.last_uid = "0"
except Exception as e:
logger.error(f"Error initializing last_uid for {self.account.email}[{self.folder_name}]: {e}")
self.last_uid = "0"
def _parse_exists_response(self, response_str):
"""Parse EXISTS response to get the message count."""
try:
parts = response_str.split()
for i, part in enumerate(parts):
if part == 'EXISTS' and i > 0:
return int(parts[i-1])
except Exception as e:
logger.debug(f"Error parsing EXISTS response '{response_str}': {e}")
return None
def _handle_new_messages(self):
"""Handle new messages detected via IDLE - end IDLE to process immediately."""
try:
message_type = "outgoing" if self.is_outgoing_folder else "incoming"
logger.info(f"New {message_type} messages detected for {self.account.email}[{self.folder_name}] - ending IDLE to process")
self._has_pending_messages = True
# End IDLE early so we can process messages immediately
try:
self.imap.send(b'DONE\r\n')
logger.debug(f"Sent DONE to end IDLE early for {self.account.email}[{self.folder_name}]")
except Exception as e:
logger.debug(f"Error sending DONE for {self.account.email}[{self.folder_name}]: {e}")
except Exception as e:
logger.error(f"Error handling new messages for {self.account.email}[{self.folder_name}]: {e}")
def _process_pending_messages(self):
"""Process any pending new messages after IDLE has ended."""
if not self._has_pending_messages:
return
try:
message_type = "outgoing" if self.is_outgoing_folder else "incoming"
logger.info(f"Processing pending {message_type} messages for {self.account.email}[{self.folder_name}]")
new_messages = self._get_new_messages_since_last()
for message_details in new_messages:
self._send_webhook_for_message(message_details)
self._has_pending_messages = False
except Exception as e:
logger.error(f"Error processing pending messages for {self.account.email}[{self.folder_name}]: {e}")
def _get_new_messages_since_last(self):
"""Get only new messages since the last check."""
try:
if self.last_uid and self.last_uid != "0":
search_criteria = f'UID {int(self.last_uid)+1}:*'
result, messages = self.imap.uid('search', None, search_criteria)
else:
result, messages = self.imap.uid('search', None, 'ALL')
if result != 'OK' or not messages[0]:
return []
new_uids = messages[0].split()
if not new_uids:
return []
# Limit to last 5 new messages to avoid webhook spam
recent_uids = new_uids[-5:] if len(new_uids) > 5 else new_uids
new_messages = []
for uid in recent_uids:
details = self._get_message_details_by_uid(uid)
if details:
new_messages.append(details)
# Update last seen UID
if new_uids:
self.last_uid = new_uids[-1].decode() if isinstance(new_uids[-1], bytes) else str(new_uids[-1])
logger.debug(f"Updated last_uid for {self.account.email}[{self.folder_name}]: {self.last_uid}")
logger.info(f"Found {len(new_messages)} new messages for {self.account.email}[{self.folder_name}]")
return new_messages
except Exception as e:
logger.error(f"Error getting new messages for {self.account.email}[{self.folder_name}]: {e}")
return []
def _get_message_details_by_uid(self, uid):
"""Fetch message details by UID including body."""
try:
result, msg_data = self.imap.uid('fetch', uid, '(RFC822)')
if result != 'OK' or not msg_data or not msg_data[0]:
return None
raw_email = msg_data[0][1]
email_message = email.message_from_bytes(raw_email)
# Extract the body
body_text = self._extract_body(email_message)
details = {
"message_id": uid.decode() if isinstance(uid, bytes) else str(uid),
"uid": uid.decode() if isinstance(uid, bytes) else str(uid),
"subject": self._decode_header(email_message.get('Subject', '')),
"sender": self._decode_header(email_message.get('From', '')),
"recipient": self._decode_header(email_message.get('To', '')),
"date": email_message.get('Date', ''),
"message_id_header": email_message.get('Message-ID', ''),
"body": body_text,
"content_type": email_message.get_content_type(),
"is_multipart": email_message.is_multipart(),
}
logger.debug(f"Fetched message UID {uid} for {self.account.email}[{self.folder_name}]: {details['subject']}")
return details
except Exception as e:
logger.error(f"Error fetching message UID {uid} for {self.account.email}[{self.folder_name}]: {e}")
return None
def _extract_body(self, email_message):
"""Extract the text body from an email message."""
body = ""
try:
if email_message.is_multipart():
# Handle multipart messages (most common)
for part in email_message.walk():
content_type = part.get_content_type()
content_disposition = str(part.get("Content-Disposition", ""))
# Skip attachments
if "attachment" in content_disposition:
continue
# Get text parts
if content_type == "text/plain":
charset = part.get_content_charset() or 'utf-8'
part_body = part.get_payload(decode=True)
if part_body:
body += part_body.decode(charset, errors='replace') + "\n"
elif content_type == "text/html" and not body:
# Use HTML as fallback if no plain text
charset = part.get_content_charset() or 'utf-8'
part_body = part.get_payload(decode=True)
if part_body:
html_body = part_body.decode(charset, errors='replace')
# Simple HTML to text conversion
body = self._html_to_text(html_body)
else:
# Handle single-part messages
content_type = email_message.get_content_type()
if content_type in ["text/plain", "text/html"]:
charset = email_message.get_content_charset() or 'utf-8'
payload = email_message.get_payload(decode=True)
if payload:
if content_type == "text/html":
body = self._html_to_text(payload.decode(charset, errors='replace'))
else:
body = payload.decode(charset, errors='replace')
except Exception as e:
logger.error(f"Error extracting email body: {e}")
body = "[Error extracting message body]"
return body.strip()
def _html_to_text(self, html):
"""Convert HTML to plain text (basic conversion)."""
try:
import re
# Remove HTML tags
text = re.sub('<[^<]+?>', '', html)
# Decode HTML entities
import html as html_module
text = html_module.unescape(text)
# Clean up whitespace
text = re.sub(r'\n\s*\n', '\n\n', text) # Multiple newlines to double
text = re.sub(r'[ \t]+', ' ', text) # Multiple spaces to single
return text.strip()
except Exception as e:
logger.error(f"Error converting HTML to text: {e}")
return html # Return original HTML if conversion fails
def _decode_header(self, header_value):
"""Decode email header values that might be encoded."""
if not header_value:
return ""
try:
from email.header import decode_header
decoded_parts = decode_header(header_value)
decoded_string = ""
for part, encoding in decoded_parts:
if isinstance(part, bytes):
decoded_string += part.decode(encoding or 'utf-8', errors='replace')
else:
decoded_string += part
return decoded_string.strip()
except Exception as e:
logger.warning(f"Error decoding header '{header_value}': {e}")
return str(header_value).strip()
def _send_webhook_for_message(self, message_details):
"""Send webhook notification for a specific message."""
try:
# Determine event type based on folder
event_type = "sent_message" if self.is_outgoing_folder else "new_message"
webhook_data = {
"email": self.account.email,
"event": event_type,
"folder": self.folder_name,
"message_id": message_details.get("message_id", "unknown"),
"uid": message_details.get("uid", "unknown"),
"subject": message_details.get("subject", ""),
"sender": message_details.get("sender", ""),
"recipient": message_details.get("recipient", ""),
"body": message_details.get("body", ""),
"content_type": message_details.get("content_type", ""),
"timestamp": timezone.now().isoformat(),
"message_timestamp": message_details.get("date", ""),
"message_id_header": message_details.get("message_id_header", ""),
"is_outgoing": self.is_outgoing_folder,
}
self._send_webhook(webhook_data)
except Exception as e:
logger.error(f"Error sending webhook for message {message_details.get('uid', 'unknown')} for {self.account.email}[{self.folder_name}]: {e}")
def _send_webhook(self, data):
try:
response = requests.post(
self.account.webhook_url,
json=data,
headers={"Content-Type": "application/json"},
timeout=10
)
direction = "outgoing" if data.get('is_outgoing') else "incoming"
if response.status_code == 200:
logger.info(f"{direction.title()} webhook sent successfully for {self.account.email}[{self.folder_name}]: {data.get('subject', 'Unknown subject')}")
else:
logger.error(f"{direction.title()} webhook failed for {self.account.email}[{self.folder_name}]: HTTP {response.status_code}")
except Exception as e:
logger.error(f"❌ Webhook error for {self.account.email}[{self.folder_name}]: {e}")
def _update_account_activity(self):
try:
from relay.models import IMAPAccount
IMAPAccount.objects.filter(email=self.account.email).update(
last_activity=timezone.now()
)
except Exception as e:
logger.error(f"Error updating activity for {self.account.email}[{self.folder_name}]: {e}")
def _update_account_status(self, is_active):
try:
from relay.models import IMAPAccount
IMAPAccount.objects.filter(email=self.account.email).update(
is_active=is_active
)
except Exception as e:
logger.error(f"Error updating status for {self.account.email}[{self.folder_name}]: {e}")
# Global connection manager instance
connection_manager = IMAPConnectionManager()

View File

@ -0,0 +1,15 @@
from rest_framework import permissions
from relay.utils.authentication import APITokenUser
class HasValidAPIToken(permissions.BasePermission):
"""
Custom permission to only allow access to API token authenticated users.
"""
def has_permission(self, request, view):
# Check if user is authenticated via API token
return (
request.user and
isinstance(request.user, APITokenUser) and
request.user.is_authenticated
)

101
src/relay/views.py Normal file
View File

@ -0,0 +1,101 @@
from rest_framework import viewsets, status
from rest_framework.decorators import action, api_view, authentication_classes, permission_classes
from rest_framework.response import Response
from rest_framework.permissions import BasePermission
from django.shortcuts import get_object_or_404
from django.utils import timezone
from relay.models import IMAPAccount
from relay.serializers import (
IMAPAccountSerializer,
IMAPAccountCreateSerializer,
IMAPAccountUpdateSerializer
)
from relay.utils.imap_manager import connection_manager
from relay.utils.authentication import APITokenAuthentication, APITokenUser
import logging
logger = logging.getLogger(__name__)
class HasValidAPIToken(BasePermission):
"""Custom permission for API token authentication."""
def has_permission(self, request, view):
return (
request.user and
isinstance(request.user, APITokenUser) and
getattr(request.user, 'is_authenticated', False)
)
class IMAPAccountViewSet(viewsets.ModelViewSet):
queryset = IMAPAccount.objects.all()
authentication_classes = [APITokenAuthentication]
permission_classes = [HasValidAPIToken]
lookup_field = 'email'
def get_serializer_class(self):
if self.action == 'create':
return IMAPAccountCreateSerializer
elif self.action in ['update', 'partial_update']:
return IMAPAccountUpdateSerializer
return IMAPAccountSerializer
def create(self, request):
logger.info(f"Create request from user: {request.user}")
logger.info(f"User type: {type(request.user)}")
logger.info(f"Is authenticated: {getattr(request.user, 'is_authenticated', False)}")
serializer = self.get_serializer(data=request.data)
if serializer.is_valid():
account = serializer.save()
# Add to connection manager
connection_manager.add_account(account)
response_serializer = IMAPAccountSerializer(account)
return Response(response_serializer.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def update(self, request, email=None):
account = get_object_or_404(IMAPAccount, email=email)
serializer = self.get_serializer(account, data=request.data, partial=True)
if serializer.is_valid():
updated_account = serializer.save()
# Update connection manager
connection_manager.update_account(email, updated_account)
response_serializer = IMAPAccountSerializer(updated_account)
return Response(response_serializer.data)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def destroy(self, request, email=None):
account = get_object_or_404(IMAPAccount, email=email)
# Remove from connection manager
connection_manager.remove_account(email)
account.delete()
return Response(
{"detail": f"Account {email} unregistered successfully"},
status=status.HTTP_200_OK
)
@action(detail=False, methods=['get'])
def health(self, request):
return Response({
"status": "healthy",
"active_connections": len(connection_manager.connections),
"manager_running": connection_manager.running
})
@api_view(['GET'])
@authentication_classes([APITokenAuthentication])
@permission_classes([HasValidAPIToken])
def test_auth(request):
return Response({
"message": "Authentication successful!",
"user": str(request.user),
"user_type": type(request.user).__name__,
"is_authenticated": getattr(request.user, 'is_authenticated', False),
"timestamp": timezone.now().isoformat()
})

11
start_dev.sh Normal file
View File

@ -0,0 +1,11 @@
#!/bin/bash
echo "Starting Django..."
python src/manage.py runserver 8009 &
echo "Waiting for Django to start..."
sleep 5
echo "Starting IMAP connections..."
python src/manage.py start_imap_connections
wait