F009 - AI Scoring and Evaluation System

// packages/manuscript-analysis/src/scoring/scoring-engine.ts
import { Logger } from '@mystoryflow/logger'
import { MetricsCollector } from '@mystoryflow/metrics'
import { ReferenceDataService } from '@mystoryflow/reference-data'
 
export class ScoringEngine {
  private logger: Logger
  private metrics: MetricsCollector
  private referenceData: ReferenceDataService
  private scoringWeights: ScoringWeights
 
  constructor() {
    this.logger = new Logger('ScoringEngine')
    this.metrics = new MetricsCollector('scoring')
    this.referenceData = new ReferenceDataService()
    this.scoringWeights = this.loadScoringWeights()
  }
 
  async calculateManuscriptScore(
    analysis: ComprehensiveAnalysis,
    metadata: ManuscriptMetadata
  ): Promise<ManuscriptScore> {
    const startTime = Date.now()
 
    // Calculate dimension scores
    const dimensionScores = await this.calculateDimensionScores(analysis)
    
    // Apply genre-specific adjustments
    const adjustedScores = await this.applyGenreAdjustments(
      dimensionScores,
      metadata.genre
    )
    
    // Calculate composite scores
    const compositeScore = this.calculateCompositeScore(adjustedScores)
    
    // Determine market readiness
    const marketReadiness = await this.assessMarketReadiness(
      adjustedScores,
      analysis.specialized.marketAnalysis
    )
    
    // Generate percentile rankings
    const percentileRanks = await this.calculatePercentileRanks(
      adjustedScores,
      metadata.genre
    )
 
    const score: ManuscriptScore = {
      overall: compositeScore,
      dimensions: adjustedScores,
      marketReadiness,
      percentileRanks,
      strengths: this.identifyStrengths(adjustedScores),
      weaknesses: this.identifyWeaknesses(adjustedScores),
      improvementPotential: this.calculateImprovementPotential(adjustedScores),
      metadata: {
        scoringVersion: '2.0',
        calculatedAt: new Date(),
        processingTime: Date.now() - startTime
      }
    }
 
    // Track metrics
    this.metrics.recordHistogram('scoring.calculation_time', Date.now() - startTime)
    this.metrics.recordGauge('scoring.overall_score', compositeScore)
 
    return score
  }
 
  private async calculateDimensionScores(
    analysis: ComprehensiveAnalysis
  ): Promise<DimensionScores> {
    const scores: DimensionScores = {
      characterDevelopment: await this.scoreCharacterDevelopment(analysis),
      plotStructure: await this.scorePlotStructure(analysis),
      writingStyle: await this.scoreWritingStyle(analysis),
      dialogue: await this.scoreDialogue(analysis),
      pacing: await this.scorePacing(analysis),
      worldBuilding: await this.scoreWorldBuilding(analysis),
      emotionalImpact: await this.scoreEmotionalImpact(analysis),
      originality: await this.scoreOriginality(analysis),
      technicalExecution: await this.scoreTechnicalExecution(analysis)
    }
 
    return scores
  }
 
  private async scoreCharacterDevelopment(
    analysis: ComprehensiveAnalysis
  ): Promise<DimensionScore> {
    const factors = {
      depth: this.evaluateCharacterDepth(analysis.overall.analysis),
      consistency: this.evaluateCharacterConsistency(analysis.chapters),
      growth: this.evaluateCharacterGrowth(analysis.overall.analysis),
      relationships: this.evaluateCharacterRelationships(analysis.overall.analysis),
      memorability: this.evaluateCharacterMemorability(analysis.overall.analysis)
    }
 
    const weightedScore = this.calculateWeightedScore(factors, {
      depth: 0.25,
      consistency: 0.20,
      growth: 0.25,
      relationships: 0.15,
      memorability: 0.15
    })
 
    return {
      score: weightedScore,
      factors,
      feedback: this.generateCharacterFeedback(factors),
      benchmarks: await this.getGenreBenchmarks('character', analysis.metadata.genre)
    }
  }
 
  private async scoreWritingStyle(
    analysis: ComprehensiveAnalysis
  ): Promise<DimensionScore> {
    const styleMetrics = {
      clarity: this.assessClarity(analysis.overall.analysis.style),
      voice: this.assessVoiceStrength(analysis.overall.analysis.style),
      imagery: this.assessImageryEffectiveness(analysis.overall.analysis.style),
      flow: this.assessProseFlow(analysis.overall.analysis.style),
      genreAppropriateness: this.assessGenreFit(
        analysis.overall.analysis.style,
        analysis.metadata.genre
      )
    }
 
    const technicalMetrics = {
      grammarAccuracy: analysis.overall.analysis.technicalScores?.grammar || 0.9,
      spellingAccuracy: analysis.overall.analysis.technicalScores?.spelling || 0.95,
      punctuationCorrectness: analysis.overall.analysis.technicalScores?.punctuation || 0.9,
      sentenceVariety: this.calculateSentenceVariety(analysis.chapters)
    }
 
    const combinedScore = this.calculateWeightedScore({
      ...styleMetrics,
      ...technicalMetrics
    }, {
      clarity: 0.20,
      voice: 0.20,
      imagery: 0.15,
      flow: 0.15,
      genreAppropriateness: 0.10,
      grammarAccuracy: 0.08,
      spellingAccuracy: 0.04,
      punctuationCorrectness: 0.04,
      sentenceVariety: 0.04
    })
 
    return {
      score: combinedScore,
      factors: { style: styleMetrics, technical: technicalMetrics },
      feedback: this.generateStyleFeedback(styleMetrics, technicalMetrics),
      benchmarks: await this.getGenreBenchmarks('style', analysis.metadata.genre)
    }
  }
}

// packages/manuscript-analysis/src/scoring/genre-adjustments.ts
export class GenreAdjustmentService {
  private genreWeights = {
    romance: {
      emotionalImpact: 1.3,
      characterDevelopment: 1.2,
      dialogue: 1.1,
      plotStructure: 0.9,
      worldBuilding: 0.7
    },
    mystery: {
      plotStructure: 1.4,
      pacing: 1.3,
      originality: 1.1,
      emotionalImpact: 0.8,
      worldBuilding: 0.9
    },
    fantasy: {
      worldBuilding: 1.5,
      originality: 1.3,
      plotStructure: 1.1,
      characterDevelopment: 1.0,
      dialogue: 0.9
    },
    literary: {
      writingStyle: 1.4,
      characterDevelopment: 1.3,
      emotionalImpact: 1.2,
      originality: 1.1,
      pacing: 0.8
    },
    thriller: {
      pacing: 1.5,
      plotStructure: 1.3,
      technicalExecution: 1.1,
      emotionalImpact: 0.9,
      worldBuilding: 0.7
    }
  }
 
  adjustScoresForGenre(
    scores: DimensionScores,
    genre: string
  ): AdjustedDimensionScores {
    const weights = this.genreWeights[genre] || {}
    const adjusted: AdjustedDimensionScores = {}
 
    for (const [dimension, score] of Object.entries(scores)) {
      const weight = weights[dimension] || 1.0
      
      adjusted[dimension] = {
        ...score,
        originalScore: score.score,
        adjustedScore: Math.min(1.0, score.score * weight),
        genreWeight: weight,
        genreImportance: this.getGenreImportance(dimension, genre)
      }
    }
 
    return adjusted
  }
 
  private getGenreImportance(dimension: string, genre: string): string {
    const importance = {
      high: 1.3,
      medium: 1.0,
      low: 0.7
    }
 
    const weight = this.genreWeights[genre]?.[dimension] || 1.0
 
    if (weight >= importance.high) return 'critical'
    if (weight >= importance.medium) return 'important'
    return 'secondary'
  }
}

// packages/manuscript-analysis/src/scoring/market-readiness.ts
import { MarketDataService } from '@mystoryflow/market-data'
 
export class MarketReadinessCalculator {
  private marketData: MarketDataService
  private thresholds: MarketThresholds
 
  constructor() {
    this.marketData = new MarketDataService()
    this.thresholds = this.loadMarketThresholds()
  }
 
  async assessMarketReadiness(
    scores: AdjustedDimensionScores,
    marketAnalysis: MarketAnalysis,
    metadata: ManuscriptMetadata
  ): Promise<MarketReadinessScore> {
    // Get current market trends
    const marketTrends = await this.marketData.getCurrentTrends(metadata.genre)
    
    // Calculate sub-scores
    const qualityScore = this.calculateQualityScore(scores)
    const marketFitScore = this.calculateMarketFit(marketAnalysis, marketTrends)
    const competitiveScore = await this.calculateCompetitivePosition(
      scores,
      metadata.genre
    )
    const commercialViability = this.assessCommercialViability(
      qualityScore,
      marketFitScore,
      metadata
    )
 
    // Determine readiness level
    const readinessLevel = this.determineReadinessLevel({
      qualityScore,
      marketFitScore,
      competitiveScore,
      commercialViability
    })
 
    return {
      overallReadiness: readinessLevel.score,
      level: readinessLevel.level, // 'ready', 'nearly-ready', 'needs-work', 'not-ready'
      components: {
        quality: {
          score: qualityScore,
          threshold: this.thresholds.quality[metadata.genre] || 0.75,
          feedback: this.getQualityFeedback(qualityScore, scores)
        },
        marketFit: {
          score: marketFitScore,
          threshold: this.thresholds.marketFit[metadata.genre] || 0.70,
          trendAlignment: marketTrends.alignment,
          feedback: this.getMarketFitFeedback(marketFitScore, marketAnalysis)
        },
        competitive: {
          score: competitiveScore,
          percentile: await this.getMarketPercentile(competitiveScore, metadata.genre),
          comparables: marketAnalysis.comparableTitles,
          feedback: this.getCompetitiveFeedback(competitiveScore)
        },
        commercial: {
          score: commercialViability,
          projectedAudience: this.estimateAudienceSize(marketAnalysis),
          revenuePoential: this.estimateRevenuePotential(commercialViability),
          feedback: this.getCommercialFeedback(commercialViability)
        }
      },
      recommendations: this.generateMarketReadinessRecommendations(readinessLevel),
      nextSteps: this.suggestNextSteps(readinessLevel, scores)
    }
  }
 
  private calculateQualityScore(scores: AdjustedDimensionScores): number {
    const criticalDimensions = [
      'characterDevelopment',
      'plotStructure',
      'writingStyle',
      'technicalExecution'
    ]
 
    const criticalScores = criticalDimensions.map(
      dim => scores[dim]?.adjustedScore || 0
    )
 
    // Quality requires all critical dimensions to be strong
    const minScore = Math.min(...criticalScores)
    const avgScore = criticalScores.reduce((a, b) => a + b, 0) / criticalScores.length
 
    // Weighted combination favoring minimum score
    return minScore * 0.4 + avgScore * 0.6
  }
 
  private determineReadinessLevel(components: {
    qualityScore: number
    marketFitScore: number
    competitiveScore: number
    commercialViability: number
  }): ReadinessLevel {
    const { qualityScore, marketFitScore, competitiveScore, commercialViability } = components
    
    // All components must meet thresholds
    if (qualityScore >= 0.85 && marketFitScore >= 0.80 && 
        competitiveScore >= 0.75 && commercialViability >= 0.70) {
      return { level: 'ready', score: 0.90 }
    }
    
    if (qualityScore >= 0.75 && marketFitScore >= 0.70 && 
        competitiveScore >= 0.65 && commercialViability >= 0.60) {
      return { level: 'nearly-ready', score: 0.70 }
    }
    
    if (qualityScore >= 0.65 && marketFitScore >= 0.60) {
      return { level: 'needs-work', score: 0.50 }
    }
    
    return { level: 'not-ready', score: 0.30 }
  }
}

// packages/manuscript-analysis/src/scoring/comparative-analysis.ts
export class ComparativeAnalysisEngine {
  private benchmarkData: BenchmarkDataService
  private similarityEngine: SimilarityEngine
 
  async generateComparativeAnalysis(
    manuscriptScore: ManuscriptScore,
    metadata: ManuscriptMetadata
  ): Promise<ComparativeAnalysis> {
    // Get benchmark data for genre
    const genreBenchmarks = await this.benchmarkData.getGenreBenchmarks(
      metadata.genre
    )
    
    // Find similar successful titles
    const comparables = await this.findComparableTitles(
      manuscriptScore,
      metadata
    )
    
    // Calculate percentile ranks
    const percentileRanks = await this.calculatePercentileRanks(
      manuscriptScore,
      genreBenchmarks
    )
    
    // Identify standout features
    const standoutFeatures = this.identifyStandoutFeatures(
      manuscriptScore,
      genreBenchmarks
    )
    
    // Generate improvement paths
    const improvementPaths = this.generateImprovementPaths(
      manuscriptScore,
      comparables
    )
 
    return {
      genreComparison: {
        benchmarks: genreBenchmarks,
        performance: this.compareToGenre(manuscriptScore, genreBenchmarks),
        percentiles: percentileRanks
      },
      similarTitles: {
        comparables,
        similarities: await this.analyzeSimilarities(manuscriptScore, comparables),
        differences: await this.analyzeDifferences(manuscriptScore, comparables)
      },
      competitivePosition: {
        strengths: standoutFeatures.strengths,
        advantages: standoutFeatures.uniqueElements,
        marketGaps: await this.identifyMarketGaps(manuscriptScore, metadata)
      },
      improvementStrategy: {
        priorityAreas: improvementPaths.priorities,
        quickWins: improvementPaths.quickWins,
        longTermGoals: improvementPaths.longTerm,
        estimatedImpact: this.estimateImprovementImpact(improvementPaths)
      }
    }
  }
 
  private async findComparableTitles(
    score: ManuscriptScore,
    metadata: ManuscriptMetadata
  ): Promise<ComparableTitle[]> {
    // Use embeddings to find similar published works
    const embedding = await this.generateScoreEmbedding(score)
    
    const similarWorks = await this.similarityEngine.findSimilar(
      embedding,
      {
        genre: metadata.genre,
        limit: 10,
        minSimilarity: 0.75
      }
    )
 
    return similarWorks.map(work => ({
      title: work.title,
      author: work.author,
      similarityScore: work.similarity,
      successMetrics: work.metrics,
      keyFeatures: work.features,
      publishedDate: work.publishedDate
    }))
  }
 
  private generateImprovementPaths(
    currentScore: ManuscriptScore,
    comparables: ComparableTitle[]
  ): ImprovementPaths {
    const successfulFeatures = this.extractSuccessfulFeatures(comparables)
    const gaps = this.identifyGaps(currentScore, successfulFeatures)
    
    return {
      priorities: gaps
        .filter(gap => gap.impact === 'high')
        .map(gap => ({
          dimension: gap.dimension,
          currentScore: gap.currentScore,
          targetScore: gap.targetScore,
          estimatedEffort: gap.effort,
          specificActions: gap.actions
        })),
      
      quickWins: gaps
        .filter(gap => gap.effort === 'low' && gap.impact !== 'low')
        .map(gap => ({
          action: gap.actions[0],
          dimension: gap.dimension,
          expectedImprovement: gap.expectedImprovement
        })),
      
      longTerm: gaps
        .filter(gap => gap.effort === 'high' && gap.impact === 'high')
        .map(gap => ({
          goal: gap.description,
          milestones: gap.milestones,
          timeframe: gap.estimatedTime
        }))
    }
  }
}

// apps/story-analyzer/src/app/api/manuscripts/[id]/score/route.ts
import { NextRequest, NextResponse } from 'next/server'
import { ScoringEngine } from '@mystoryflow/manuscript-analysis'
import { withAuth } from '@mystoryflow/auth'
import { getAnalysisResults } from '@/lib/analysis'
 
export async function GET(
  req: NextRequest,
  { params }: { params: { id: string } }
) {
  try {
    const session = await withAuth(req)
    if (!session) {
      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
    }
 
    // Get analysis results
    const analysis = await getAnalysisResults(params.id, session.user.id)
    if (!analysis) {
      return NextResponse.json(
        { error: 'Analysis not found' },
        { status: 404 }
      )
    }
 
    // Calculate comprehensive score
    const scoringEngine = new ScoringEngine()
    const score = await scoringEngine.calculateManuscriptScore(
      analysis,
      analysis.metadata
    )
 
    // Generate comparative analysis
    const comparativeEngine = new ComparativeAnalysisEngine()
    const comparison = await comparativeEngine.generateComparativeAnalysis(
      score,
      analysis.metadata
    )
 
    // Format for visualization
    const visualizationData = {
      score,
      comparison,
      charts: {
        radar: formatRadarChartData(score.dimensions),
        percentile: formatPercentileChart(score.percentileRanks),
        improvement: formatImprovementChart(comparison.improvementStrategy),
        timeline: formatProgressTimeline(analysis.history)
      }
    }
 
    return NextResponse.json(visualizationData)
  } catch (error) {
    console.error('Score calculation failed:', error)
    return NextResponse.json(
      { error: 'Failed to calculate score' },
      { status: 500 }
    )
  }
}

-- Manuscript scores
CREATE TABLE analyzer.manuscript_scores (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  manuscript_id UUID REFERENCES analyzer.manuscripts(id),
  analysis_id UUID REFERENCES analyzer.analysis_results(id),
  overall_score DECIMAL(3,2),
  dimension_scores JSONB,
  market_readiness JSONB,
  percentile_ranks JSONB,
  strengths TEXT[],
  weaknesses TEXT[],
  improvement_potential DECIMAL(3,2),
  scoring_version VARCHAR(10),
  created_at TIMESTAMP DEFAULT NOW(),
  
  INDEX idx_scores_manuscript (manuscript_id),
  INDEX idx_scores_overall (overall_score DESC)
);
 
-- Genre benchmarks
CREATE TABLE analyzer.genre_benchmarks (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  genre VARCHAR(50),
  dimension VARCHAR(50),
  percentile_25 DECIMAL(3,2),
  percentile_50 DECIMAL(3,2),
  percentile_75 DECIMAL(3,2),
  percentile_90 DECIMAL(3,2),
  sample_size INTEGER,
  last_updated TIMESTAMP DEFAULT NOW(),
  
  UNIQUE(genre, dimension),
  INDEX idx_benchmarks_genre (genre)
);
 
-- Comparative analysis cache
CREATE TABLE analyzer.comparative_analyses (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  manuscript_id UUID REFERENCES analyzer.manuscripts(id),
  score_id UUID REFERENCES analyzer.manuscript_scores(id),
  comparable_titles JSONB,
  competitive_position JSONB,
  improvement_strategy JSONB,
  market_gaps JSONB,
  created_at TIMESTAMP DEFAULT NOW(),
  expires_at TIMESTAMP,
  
  INDEX idx_comparative_manuscript (manuscript_id)
);
 
-- Score history for tracking improvements
CREATE TABLE analyzer.score_history (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  manuscript_id UUID REFERENCES analyzer.manuscripts(id),
  version INTEGER,
  overall_score DECIMAL(3,2),
  dimension_changes JSONB,
  improvement_from_previous DECIMAL(3,2),
  created_at TIMESTAMP DEFAULT NOW(),
  
  INDEX idx_history_manuscript (manuscript_id, version DESC)
);