Zetian.Storage.MongoDB 1.0.4

Zetian.Storage.MongoDB - NoSQL Storage

MongoDB NoSQL storage provider for Zetian SMTP Server. Provides flexible, schema-less message persistence with advanced features including GridFS for large attachment storage, automatic TTL-based message expiration, horizontal sharding for scalability, text indexing for full-text search, and built-in compression. Perfect for modern applications requiring flexible data models, automatic scaling, and document-oriented storage without rigid schemas.

⚡ Features

• 🔍 Text Search - Full-text search capabilities
• 🌐 Flexible Schema - NoSQL document flexibility
• 🌍 Replica Sets - Built-in high availability support
• 🔄 Change Streams - Real-time change notifications
• 📊 Aggregation - Powerful aggregation pipeline support
• 🔀 Sharding - Horizontal scaling for massive deployments
• 🗜️ Compression - Optional GZIP compression for messages
• 🚀 High Performance - Optimized for write-heavy workloads
• 📂 GridFS Support - Automatic large file handling with GridFS
• ⏰ TTL Expiration - Automatic message cleanup after specified time

📦 Installation

# Install SMTP Server and Storage Provider
dotnet add package Zetian
dotnet add package Zetian.Storage.MongoDB

🚀 Quick Start

Basic Configuration

using Zetian.Server;
using Zetian.Storage.MongoDB.Extensions;

// Configure with connection string and database
var server = new SmtpServerBuilder()
    .Port(25)
    .WithMongoDbStorage(
        "mongodb://localhost:27017",
        "smtp_database")
    .Build();

await server.StartAsync();

Advanced Configuration

var server = new SmtpServerBuilder()
    .Port(25)
    .WithMongoDbStorage(
        "mongodb://localhost:27017",
        "smtp_database",
        config =>
        {
            config.CollectionName = "email_messages";
            config.UseGridFsForLargeMessages = true;
            config.GridFsThresholdMB = 5;
            config.EnableTTL = true;
            config.TTLDays = 30;
            config.EnableSharding = true;
            config.ShardKeyField = "received_date";
            config.CompressMessageBody = true;
            config.AutoCreateIndexes = true;
        })
    .Build();

🛠️ Configuration Options

📏 Document Schema

{
  _id: ObjectId("..."),
  messageId: "msg-123456",
  sessionId: "session-789",
  fromAddress: "sender@example.com",
  toAddresses: ["recipient1@example.com", "recipient2@example.com"],
  ccAddresses: [],
  bccAddresses: [],
  subject: "Email Subject",
  receivedDate: ISODate("2024-01-01T12:00:00Z"),
  messageSize: NumberLong(2048),
  messageBody: BinData(...),  // When small
  isCompressed: false,
  headers: {
    "From": "sender@example.com",
    "To": "recipient@example.com",
    "Subject": "Email Subject",
    "Message-ID": "<msg-123456@example.com>",
    "X-Priority": "Normal"
  },
  attachments: [
    {
      filename: "document.pdf",
      contentType: "application/pdf",
      size: 102400,
      gridFsId: ObjectId("...")  // If stored in GridFS
    }
  ],
  hasAttachments: true,
  attachmentCount: 1,
  priority: "normal",
  remoteIP: "192.168.1.100",
  isStoredInGridFs: false,
  gridFsId: null,  // ObjectId when body in GridFS
  createdAt: ISODate("2024-01-01T12:00:00Z"),
  metadata: {  // Custom metadata
    "processed": true,
    "spam_score": 0.1
  }
}

🎯 Usage Examples

Connection String Options

// Standalone server
.WithMongoDbStorage("mongodb://localhost:27017", "smtp_db")

// Replica set
.WithMongoDbStorage(
    "mongodb://node1:27017,node2:27017,node3:27017/?replicaSet=rs0", 
    "smtp_db")

// With authentication
.WithMongoDbStorage(
    "mongodb://username:password@localhost:27017/smtp_db?authSource=admin", 
    "smtp_db")

// MongoDB Atlas
.WithMongoDbStorage(
    "mongodb+srv://username:password@cluster.mongodb.net/smtp_db?retryWrites=true", 
    "smtp_db")

// With connection pooling
.WithMongoDbStorage(
    "mongodb://localhost:27017/?maxPoolSize=100&minPoolSize=10", 
    "smtp_db")

GridFS for Large Messages

// Configure GridFS
.WithMongoDbStorage(
    "mongodb://localhost:27017",
    "smtp_database",
    config =>
    {
        config.UseGridFsForLargeMessages = true;
        config.GridFsThresholdMB = 5; // Store messages > 5MB in GridFS
        config.GridFsBucketName = "email_files";
    })

// Retrieving messages with GridFS
var message = await messageStore.GetMessageAsync("msg-123");
if (message.IsStoredInGridFs)
{
    // Body automatically retrieved from GridFS
    var body = message.MessageBody;
}

TTL (Time To Live) Configuration

// Auto-delete messages after 30 days
.WithMongoDbStorage(
    "mongodb://localhost:27017",
    "smtp_database",
    config =>
    {
        config.EnableTTL = true;
        config.TTLDays = 30;
    })

// Different TTL strategies
.WithMongoDbStorage(
    "mongodb://localhost:27017",
    "smtp_database",
    config =>
    {
        config.EnableTTL = true;
        config.TTLDays = 7;   // Delete after 1 week
    })

🔍 Querying Messages

MongoDB Query Examples

// Find messages from last 7 days
db.messages.find({
  receivedDate: { $gte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
})

// Full-text search
db.messages.find({
  $text: { $search: "important meeting" }
})

// Find by sender domain
db.messages.find({
  fromAddress: { $regex: "@example\.com$" }
})

// Aggregation pipeline
db.messages.aggregate([
  { $match: { receivedDate: { $gte: ISODate("2024-01-01") } } },
  { $group: {
    _id: "$fromAddress",
    count: { $sum: 1 },
    totalSize: { $sum: "$messageSize" }
  }},
  { $sort: { count: -1 } },
  { $limit: 10 }
])

🚀 Performance Optimization

Indexing Strategy

// Create compound indexes
db.messages.createIndex({ receivedDate: -1, fromAddress: 1 })
db.messages.createIndex({ sessionId: 1, messageId: 1 })

// Text index for full-text search
db.messages.createIndex({ subject: "text", "headers.From": "text" })

// Partial index for unread messages
db.messages.createIndex(
  { receivedDate: -1 },
  { partialFilterExpression: { "metadata.read": false } }
)

// Hashed index for sharding
db.messages.createIndex({ messageId: "hashed" })

Sharding Configuration

// Enable sharding on database
sh.enableSharding("smtp_database")

// Shard the collection
sh.shardCollection(
  "smtp_database.messages",
  { receivedDate: 1 }  // Range-based sharding
)

// Or use hashed sharding for even distribution
sh.shardCollection(
  "smtp_database.messages",
  { messageId: "hashed" }
)

📊 Change Streams (Real-time)

// Watch for new messages
var pipeline = new EmptyPipelineDefinition<ChangeStreamDocument<BsonDocument>>()
    .Match(change => change.OperationType == ChangeStreamOperationType.Insert);

using (var cursor = await collection.WatchAsync(pipeline))
{
    await cursor.ForEachAsync(change =>
    {
        Console.WriteLine($"New message: {change.FullDocument["messageId"]}");
    });
}

🌐 Replica Set Configuration

// Connection to replica set
config.ConnectionString = 
    "mongodb://node1:27017,node2:27017,node3:27017/" +
    "?replicaSet=rs0" +
    "&readPreference=secondaryPreferred" +
    "&readConcernLevel=majority";

// Read preferences
config.ReadPreference = "secondaryPreferred"; // Read from secondaries when possible
config.ReadConcern = "majority"; // Read committed data

☁️ MongoDB Atlas Integration

// Atlas connection
config.ConnectionString = 
    "mongodb+srv://username:password@cluster.mongodb.net/smtp_db" +
    "?retryWrites=true&w=majority";

// Atlas Search (Lucene-based)
var searchStage = new BsonDocument("$search", new BsonDocument
{
    { "index", "message_search" },
    { "text", new BsonDocument
    {
        { "query", "important" },
        { "path", new BsonArray { "subject", "messageBody" } }
    }}
});

📈 Monitoring & Maintenance

Collection Statistics

// Get collection stats
db.messages.stats()

// Check index usage
db.messages.aggregate([
  { $indexStats: {} }
])

// Find slow queries
db.currentOp({ 
  "secs_running": { $gte: 5 },
  "ns": "smtp_database.messages"
})

Maintenance Tasks

// Compact collection
db.runCommand({ compact: "messages" })

// Rebuild indexes
db.messages.reIndex()

// Validate collection
db.messages.validate({ full: true })

// Clean up orphaned GridFS chunks
db.fs.chunks.deleteMany({
  files_id: { $nin: db.fs.files.distinct("_id") }
})

🔒 Security Best Practices

Authentication & Authorization

// Create dedicated user
db.createUser({
  user: "smtp_app",
  pwd: "secure_password",
  roles: [
    { role: "readWrite", db: "smtp_database" },
    { role: "dbAdmin", db: "smtp_database" }
  ]
})

// Enable authentication in connection string
config.ConnectionString = 
    "mongodb://smtp_app:secure_password@localhost:27017/" +
    "smtp_database?authSource=smtp_database";

Encryption

// TLS/SSL connection
config.ConnectionString = 
    "mongodb://localhost:27017/?ssl=true&" +
    "sslCertificateAuthorityFile=/path/to/ca.pem";

🔧 Troubleshooting

Common Issues

Connection Timeout

// Increase connection timeout
config.ConnectionTimeoutSeconds = 60;

Memory Issues with Large Collections

// Use cursor with batch size
db.messages.find().batchSize(100).forEach(doc => {
  // Process document
})

// Limit query results
db.messages.find().limit(1000).hint({ receivedDate: -1 })

📋 Requirements

• .NET 6.0, 7.0, 8.0, 9.0, or 10.0
• Zetian SMTP Server package
• For Atlas: MongoDB Atlas cluster
• MongoDB.Driver package (included)
• For sharding: MongoDB sharded cluster
• MongoDB 4.0 or later (4.4+ recommended)

📚 Documentation & Support

• Issues: GitHub Issues
• Examples: GitHub Examples
• Discussions: GitHub Discussions
• Documentation: Zetian Documentation
• MongoDB Docs: MongoDB Documentation

📄 License

MIT License - see LICENSE

Built with ❤️ for the .NET community