Eve library

Content Moderation Integrations

Guide

Revision 1

On this page

External content moderation service integration documentation.


Overview#

Aphrodite integrates with multiple moderation services for compliance.

Service Purpose Priority
Thorn Safer CSAM detection (required) Critical
PhotoDNA CSAM detection (backup) Critical
Hive AI Content classification High
AWS Rekognition Unsafe content detection Medium

Thorn Safer (CSAM Detection)#

CRITICAL: Legally required for CSAM detection.

Configuration#

typescript
interface ThornConfig {
  apiKey: string;
  baseUrl: string;
  webhookUrl: string;
}

Submit Content for Analysis#

typescript
async function scanContent(
  mediaUrl: string,
  contentId: string
): Promise<ScanResult> {
  const response = await fetch(`${config.baseUrl}/v1/scans`, {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${config.apiKey}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({
      media_url: mediaUrl,
      external_id: contentId,
      callback_url: config.webhookUrl,
    }),
  });

  return response.json();
}

Handle Results#

typescript
app.post('/webhooks/thorn', async (req, res) => {
  const { external_id, classification, confidence } = req.body;

  if (classification === 'csam' || classification === 'potential_csam') {
    // CRITICAL: Immediate action required
    await quarantineContent(external_id);
    await reportToNCMEC(external_id, req.body);
    await notifyCompliance(external_id, req.body);
  }

  res.status(200).send('OK');
});

PhotoDNA (Backup CSAM Detection)#

Configuration#

typescript
interface PhotoDNAConfig {
  subscriptionKey: string;
  endpoint: string;
}

Hash and Match#

typescript
async function checkPhotoHash(imageBuffer: Buffer): Promise<MatchResult> {
  const response = await fetch(
    `${config.endpoint}/contentmoderator/moderate/v1.0/ProcessImage/Match`,
    {
      method: 'POST',
      headers: {
        'Ocp-Apim-Subscription-Key': config.subscriptionKey,
        'Content-Type': 'application/octet-stream',
      },
      body: imageBuffer,
    }
  );

  return response.json();
}

Hive AI (Content Classification)#

Configuration#

typescript
interface HiveConfig {
  apiKey: string;
  baseUrl: string;
  autoRejectThreshold: number;
  autoApproveThreshold: number;
}

Classify Content#

typescript
async function classifyContent(
  mediaUrl: string
): Promise<ClassificationResult> {
  const response = await fetch(`${config.baseUrl}/v2/task/sync`, {
    method: 'POST',
    headers: {
      Authorization: `Token ${config.apiKey}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({
      url: mediaUrl,
    }),
  });

  const data = await response.json();
  return parseHiveResponse(data);
}

function parseHiveResponse(data: any): ClassificationResult {
  const output = data.status[0].response.output[0];

  return {
    safe:
      output.classes.find((c: any) => c.class === 'general_nsfw')?.score < 0.5,
    adult: output.classes.find((c: any) => c.class === 'general_nsfw')?.score,
    violence: output.classes.find((c: any) => c.class === 'violence')?.score,
    hate: output.classes.find((c: any) => c.class === 'hate')?.score,
    categories: output.classes.map((c: any) => ({
      name: c.class,
      score: c.score,
    })),
  };
}

Auto-Decision Logic#

typescript
function makeDecision(result: ClassificationResult): ModerationDecision {
  // Auto-reject high confidence violations
  if (result.violence > config.autoRejectThreshold) {
    return {
      action: 'reject',
      reason: 'violence',
      confidence: result.violence,
    };
  }

  if (result.hate > config.autoRejectThreshold) {
    return { action: 'reject', reason: 'hate_speech', confidence: result.hate };
  }

  // Auto-approve clearly safe content
  if (
    result.safe &&
    result.violence < config.autoApproveThreshold &&
    result.hate < config.autoApproveThreshold
  ) {
    return {
      action: 'approve',
      reason: 'auto_approved',
      confidence: 1 - result.violence,
    };
  }

  // Send to human review
  return { action: 'review', reason: 'uncertain', confidence: null };
}

AWS Rekognition#

Configuration#

typescript
interface RekognitionConfig {
  region: string;
  accessKeyId: string;
  secretAccessKey: string;
}

Detect Moderation Labels#

typescript
import {
  RekognitionClient,
  DetectModerationLabelsCommand,
} from '@aws-sdk/client-rekognition';

const client = new RekognitionClient({
  region: config.region,
  credentials: {
    accessKeyId: config.accessKeyId,
    secretAccessKey: config.secretAccessKey,
  },
});

async function detectModerationLabels(
  imageBytes: Buffer
): Promise<ModerationLabel[]> {
  const command = new DetectModerationLabelsCommand({
    Image: { Bytes: imageBytes },
    MinConfidence: 50,
  });

  const response = await client.send(command);
  return response.ModerationLabels || [];
}

NCMEC Reporting#

CRITICAL: Legally required in the United States.

Configuration#

typescript
interface NCMECConfig {
  espId: string;
  apiKey: string;
  reportingEnabled: boolean;
}

Submit Report#

typescript
async function reportToNCMEC(incident: CSAMIncident): Promise<string> {
  if (!config.reportingEnabled) {
    throw new Error('NCMEC reporting is disabled');
  }

  const response = await fetch(
    'https://report.cybertip.org/ispws/submitreport',
    {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${config.apiKey}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({
        espId: config.espId,
        incidentType: 'Child Sexual Abuse Material',
        incidentDateTime: incident.detectedAt,
        reportingUserId: incident.uploadedBy,
        additionalInfo: incident.details,
        preservedEvidence: {
          mediaHash: incident.mediaHash,
          storageLocation: incident.quarantineLocation,
        },
      }),
    }
  );

  const data = await response.json();
  return data.reportId;
}

Moderation Pipeline#

typescript
// libs/aphrodite/moderation/src/pipeline.ts
export class ModerationPipeline {
  async processMedia(media: Media): Promise<ModerationResult> {
    // Step 1: CSAM Detection (CRITICAL - always first)
    const csamResult = await this.thornService.scan(media.url);
    if (csamResult.classification !== 'safe') {
      await this.handleCSAMDetection(media, csamResult);
      return { status: 'quarantined', reason: 'csam_detection' };
    }

    // Step 2: Content Classification
    const classification = await this.hiveService.classify(media.url);

    // Step 3: Auto-decision
    const decision = this.makeDecision(classification);

    if (decision.action === 'approve') {
      return { status: 'approved', confidence: decision.confidence };
    }

    if (decision.action === 'reject') {
      return { status: 'rejected', reason: decision.reason };
    }

    // Step 4: Queue for human review
    await this.queueForReview(media, classification);
    return { status: 'pending_review' };
  }
}

Compliance Checklist#

  • Thorn Safer integration active
  • PhotoDNA backup integration active
  • NCMEC reporting configured and tested
  • All uploads scanned before publication
  • Quarantine system implemented
  • Audit logging enabled
  • Incident response procedures documented
  • Compliance team notification system active

Document History#

Date Version Changes
2026-01-18 1.0.0 Initial Moderation Integrations docs