External content moderation service integration documentation.
Overview#
Aphrodite integrates with multiple moderation services for compliance.
| Service | Purpose | Priority |
|---|---|---|
| Thorn Safer | CSAM detection (required) | Critical |
| PhotoDNA | CSAM detection (backup) | Critical |
| Hive AI | Content classification | High |
| AWS Rekognition | Unsafe content detection | Medium |
Thorn Safer (CSAM Detection)#
CRITICAL: Legally required for CSAM detection.
Configuration#
typescript
interface ThornConfig {
apiKey: string;
baseUrl: string;
webhookUrl: string;
}
Submit Content for Analysis#
typescript
async function scanContent(
mediaUrl: string,
contentId: string
): Promise<ScanResult> {
const response = await fetch(`${config.baseUrl}/v1/scans`, {
method: 'POST',
headers: {
Authorization: `Bearer ${config.apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
media_url: mediaUrl,
external_id: contentId,
callback_url: config.webhookUrl,
}),
});
return response.json();
}
Handle Results#
typescript
app.post('/webhooks/thorn', async (req, res) => {
const { external_id, classification, confidence } = req.body;
if (classification === 'csam' || classification === 'potential_csam') {
// CRITICAL: Immediate action required
await quarantineContent(external_id);
await reportToNCMEC(external_id, req.body);
await notifyCompliance(external_id, req.body);
}
res.status(200).send('OK');
});
PhotoDNA (Backup CSAM Detection)#
Configuration#
typescript
interface PhotoDNAConfig {
subscriptionKey: string;
endpoint: string;
}
Hash and Match#
typescript
async function checkPhotoHash(imageBuffer: Buffer): Promise<MatchResult> {
const response = await fetch(
`${config.endpoint}/contentmoderator/moderate/v1.0/ProcessImage/Match`,
{
method: 'POST',
headers: {
'Ocp-Apim-Subscription-Key': config.subscriptionKey,
'Content-Type': 'application/octet-stream',
},
body: imageBuffer,
}
);
return response.json();
}
Hive AI (Content Classification)#
Configuration#
typescript
interface HiveConfig {
apiKey: string;
baseUrl: string;
autoRejectThreshold: number;
autoApproveThreshold: number;
}
Classify Content#
typescript
async function classifyContent(
mediaUrl: string
): Promise<ClassificationResult> {
const response = await fetch(`${config.baseUrl}/v2/task/sync`, {
method: 'POST',
headers: {
Authorization: `Token ${config.apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
url: mediaUrl,
}),
});
const data = await response.json();
return parseHiveResponse(data);
}
function parseHiveResponse(data: any): ClassificationResult {
const output = data.status[0].response.output[0];
return {
safe:
output.classes.find((c: any) => c.class === 'general_nsfw')?.score < 0.5,
adult: output.classes.find((c: any) => c.class === 'general_nsfw')?.score,
violence: output.classes.find((c: any) => c.class === 'violence')?.score,
hate: output.classes.find((c: any) => c.class === 'hate')?.score,
categories: output.classes.map((c: any) => ({
name: c.class,
score: c.score,
})),
};
}
Auto-Decision Logic#
typescript
function makeDecision(result: ClassificationResult): ModerationDecision {
// Auto-reject high confidence violations
if (result.violence > config.autoRejectThreshold) {
return {
action: 'reject',
reason: 'violence',
confidence: result.violence,
};
}
if (result.hate > config.autoRejectThreshold) {
return { action: 'reject', reason: 'hate_speech', confidence: result.hate };
}
// Auto-approve clearly safe content
if (
result.safe &&
result.violence < config.autoApproveThreshold &&
result.hate < config.autoApproveThreshold
) {
return {
action: 'approve',
reason: 'auto_approved',
confidence: 1 - result.violence,
};
}
// Send to human review
return { action: 'review', reason: 'uncertain', confidence: null };
}
AWS Rekognition#
Configuration#
typescript
interface RekognitionConfig {
region: string;
accessKeyId: string;
secretAccessKey: string;
}
Detect Moderation Labels#
typescript
import {
RekognitionClient,
DetectModerationLabelsCommand,
} from '@aws-sdk/client-rekognition';
const client = new RekognitionClient({
region: config.region,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
},
});
async function detectModerationLabels(
imageBytes: Buffer
): Promise<ModerationLabel[]> {
const command = new DetectModerationLabelsCommand({
Image: { Bytes: imageBytes },
MinConfidence: 50,
});
const response = await client.send(command);
return response.ModerationLabels || [];
}
NCMEC Reporting#
CRITICAL: Legally required in the United States.
Configuration#
typescript
interface NCMECConfig {
espId: string;
apiKey: string;
reportingEnabled: boolean;
}
Submit Report#
typescript
async function reportToNCMEC(incident: CSAMIncident): Promise<string> {
if (!config.reportingEnabled) {
throw new Error('NCMEC reporting is disabled');
}
const response = await fetch(
'https://report.cybertip.org/ispws/submitreport',
{
method: 'POST',
headers: {
Authorization: `Bearer ${config.apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
espId: config.espId,
incidentType: 'Child Sexual Abuse Material',
incidentDateTime: incident.detectedAt,
reportingUserId: incident.uploadedBy,
additionalInfo: incident.details,
preservedEvidence: {
mediaHash: incident.mediaHash,
storageLocation: incident.quarantineLocation,
},
}),
}
);
const data = await response.json();
return data.reportId;
}
Moderation Pipeline#
typescript
// libs/aphrodite/moderation/src/pipeline.ts
export class ModerationPipeline {
async processMedia(media: Media): Promise<ModerationResult> {
// Step 1: CSAM Detection (CRITICAL - always first)
const csamResult = await this.thornService.scan(media.url);
if (csamResult.classification !== 'safe') {
await this.handleCSAMDetection(media, csamResult);
return { status: 'quarantined', reason: 'csam_detection' };
}
// Step 2: Content Classification
const classification = await this.hiveService.classify(media.url);
// Step 3: Auto-decision
const decision = this.makeDecision(classification);
if (decision.action === 'approve') {
return { status: 'approved', confidence: decision.confidence };
}
if (decision.action === 'reject') {
return { status: 'rejected', reason: decision.reason };
}
// Step 4: Queue for human review
await this.queueForReview(media, classification);
return { status: 'pending_review' };
}
}
Compliance Checklist#
- Thorn Safer integration active
- PhotoDNA backup integration active
- NCMEC reporting configured and tested
- All uploads scanned before publication
- Quarantine system implemented
- Audit logging enabled
- Incident response procedures documented
- Compliance team notification system active
Document History#
| Date | Version | Changes |
|---|---|---|
| 2026-01-18 | 1.0.0 | Initial Moderation Integrations docs |