F012 - Advanced Analytics and Insights

// packages/analytics-engine/src/services/advanced-analytics.ts
import { AIService } from '@mystoryflow/manuscript-analysis'
import { DataWarehouse } from '@mystoryflow/data-warehouse'
import { Logger } from '@mystoryflow/logger'
import { TimeSeriesAnalyzer } from './time-series'
import { PredictiveModeler } from './predictive-modeling'
 
export class AdvancedAnalyticsEngine {
  private aiService: AIService
  private dataWarehouse: DataWarehouse
  private timeSeriesAnalyzer: TimeSeriesAnalyzer
  private predictiveModeler: PredictiveModeler
  private logger: Logger
 
  constructor() {
    this.aiService = new AIService()
    this.dataWarehouse = new DataWarehouse()
    this.timeSeriesAnalyzer = new TimeSeriesAnalyzer()
    this.predictiveModeler = new PredictiveModeler()
    this.logger = new Logger('AdvancedAnalytics')
  }
 
  async generateComprehensiveInsights(params: {
    userId: string
    manuscriptId?: string
    timeRange: TimeRange
    insightTypes: InsightType[]
  }): Promise<ComprehensiveInsights> {
    const { userId, manuscriptId, timeRange, insightTypes } = params
 
    // Gather historical data
    const historicalData = await this.gatherHistoricalData(
      userId,
      manuscriptId,
      timeRange
    )
 
    // Generate insights based on requested types
    const insights: ComprehensiveInsights = {
      summary: await this.generateExecutiveSummary(historicalData),
      insights: {}
    }
 
    for (const type of insightTypes) {
      switch (type) {
        case 'writing-progress':
          insights.insights.writingProgress = await this.analyzeWritingProgress(
            historicalData
          )
          break
          
        case 'quality-trends':
          insights.insights.qualityTrends = await this.analyzeQualityTrends(
            historicalData
          )
          break
          
        case 'market-positioning':
          insights.insights.marketPositioning = await this.analyzeMarketPositioning(
            manuscriptId,
            historicalData
          )
          break
          
        case 'predictive-success':
          insights.insights.successPrediction = await this.predictSuccess(
            manuscriptId,
            historicalData
          )
          break
          
        case 'improvement-velocity':
          insights.insights.improvementVelocity = await this.analyzeImprovementVelocity(
            historicalData
          )
          break
          
        case 'engagement-patterns':
          insights.insights.engagementPatterns = await this.analyzeEngagementPatterns(
            historicalData
          )
          break
      }
    }
 
    // Generate actionable recommendations
    insights.recommendations = await this.generateActionableRecommendations(
      insights.insights
    )
 
    return insights
  }
 
  private async analyzeWritingProgress(
    data: HistoricalData
  ): Promise<WritingProgressInsights> {
    // Calculate writing velocity
    const velocityMetrics = this.timeSeriesAnalyzer.calculateVelocity(
      data.wordCountHistory,
      'daily'
    )
 
    // Identify productivity patterns
    const productivityPatterns = await this.identifyProductivityPatterns(
      data.writingSessions
    )
 
    // Analyze consistency
    const consistencyScore = this.calculateConsistencyScore(
      data.writingSessions
    )
 
    // Project completion timeline
    const projectedCompletion = await this.projectCompletionDate(
      data.currentWordCount,
      data.targetWordCount,
      velocityMetrics
    )
 
    // Identify blockers and accelerators
    const factors = await this.analyzeProductivityFactors(data)
 
    return {
      currentVelocity: velocityMetrics.current,
      averageVelocity: velocityMetrics.average,
      trend: velocityMetrics.trend,
      productivityPatterns,
      consistencyScore,
      projectedCompletion,
      accelerators: factors.accelerators,
      blockers: factors.blockers,
      recommendations: this.generateProgressRecommendations(
        velocityMetrics,
        productivityPatterns
      )
    }
  }
 
  private async analyzeQualityTrends(
    data: HistoricalData
  ): Promise<QualityTrendsInsights> {
    // Extract quality scores over time
    const qualityTimeSeries = data.analyses.map(analysis => ({
      timestamp: analysis.createdAt,
      scores: analysis.scores,
      overallScore: analysis.overallScore
    }))
 
    // Analyze dimension-specific trends
    const dimensionTrends = await this.analyzeDimensionTrends(qualityTimeSeries)
 
    // Identify improvement areas
    const improvementAreas = this.identifyImprovementAreas(dimensionTrends)
 
    // Calculate improvement velocity
    const improvementVelocity = this.timeSeriesAnalyzer.calculateTrend(
      qualityTimeSeries.map(q => ({ 
        timestamp: q.timestamp, 
        value: q.overallScore 
      }))
    )
 
    // Predict quality trajectory
    const qualityPrediction = await this.predictiveModeler.predictTimeSeries(
      qualityTimeSeries,
      30 // 30 days forecast
    )
 
    return {
      currentQuality: qualityTimeSeries[qualityTimeSeries.length - 1]?.overallScore || 0,
      qualityTrend: improvementVelocity,
      dimensionTrends,
      strongestDimensions: this.getTopDimensions(dimensionTrends, 'positive'),
      weakestDimensions: this.getTopDimensions(dimensionTrends, 'negative'),
      improvementAreas,
      prediction: qualityPrediction,
      benchmarkComparison: await this.compareToGenreBenchmarks(
        qualityTimeSeries[qualityTimeSeries.length - 1],
        data.genre
      )
    }
  }
 
  private async analyzeMarketPositioning(
    manuscriptId: string,
    data: HistoricalData
  ): Promise<MarketPositioningInsights> {
    // Get current market data
    const marketData = await this.dataWarehouse.getMarketData(data.genre)
 
    // Analyze competitive positioning
    const competitiveAnalysis = await this.analyzeCompetitivePosition(
      manuscriptId,
      marketData
    )
 
    // Identify market trends
    const marketTrends = await this.identifyMarketTrends(
      data.genre,
      marketData
    )
 
    // Calculate market fit score
    const marketFitScore = await this.calculateMarketFit(
      data.latestAnalysis,
      marketTrends
    )
 
    // Find market opportunities
    const opportunities = await this.identifyMarketOpportunities(
      data.latestAnalysis,
      marketData,
      marketTrends
    )
 
    return {
      competitivePosition: competitiveAnalysis,
      marketFitScore,
      genreRanking: competitiveAnalysis.percentileRank,
      trendingElements: marketTrends.trending,
      decliningElements: marketTrends.declining,
      opportunities,
      comparableTitles: await this.findComparableTitles(
        manuscriptId,
        marketData
      ),
      uniqueSellingPoints: await this.identifyUSPs(
        data.latestAnalysis,
        marketData
      ),
      recommendations: this.generateMarketRecommendations(
        marketFitScore,
        opportunities
      )
    }
  }
 
  private async predictSuccess(
    manuscriptId: string,
    data: HistoricalData
  ): Promise<SuccessPredictionInsights> {
    // Gather all relevant features
    const features = await this.extractPredictiveFeatures(data)
 
    // Run multiple prediction models
    const predictions = {
      commercial: await this.predictiveModeler.predictCommercialSuccess(features),
      critical: await this.predictiveModeler.predictCriticalSuccess(features),
      reader: await this.predictiveModeler.predictReaderEngagement(features),
      viral: await this.predictiveModeler.predictViralPotential(features)
    }
 
    // Identify key success factors
    const successFactors = await this.identifyKeySuccessFactors(
      features,
      predictions
    )
 
    // Calculate confidence intervals
    const confidence = this.calculatePredictionConfidence(
      features,
      data.sampleSize
    )
 
    // Generate scenario analysis
    const scenarios = await this.generateScenarioAnalysis(
      manuscriptId,
      features,
      predictions
    )
 
    return {
      predictions,
      confidence,
      successFactors,
      riskFactors: this.identifyRiskFactors(features, predictions),
      scenarios,
      improvementImpact: await this.simulateImprovementImpact(
        features,
        predictions
      ),
      timeline: this.generateSuccessTimeline(predictions),
      recommendations: this.generateSuccessRecommendations(
        predictions,
        successFactors
      )
    }
  }
}

// packages/analytics-engine/src/services/time-series.ts
import { DataFrame } from '@mystoryflow/data-processing'
 
export class TimeSeriesAnalyzer {
  calculateVelocity(
    data: TimeSeriesData[],
    granularity: 'daily' | 'weekly' | 'monthly'
  ): VelocityMetrics {
    // Aggregate data by granularity
    const aggregated = this.aggregateByPeriod(data, granularity)
    
    // Calculate moving averages
    const ma7 = this.movingAverage(aggregated, 7)
    const ma30 = this.movingAverage(aggregated, 30)
    
    // Calculate velocity
    const currentVelocity = this.calculateCurrentVelocity(aggregated)
    const averageVelocity = aggregated.reduce((sum, d) => sum + d.value, 0) / 
                           aggregated.length
    
    // Determine trend
    const trend = this.determineTrend(ma7, ma30)
    
    // Calculate acceleration
    const acceleration = this.calculateAcceleration(aggregated)
 
    return {
      current: currentVelocity,
      average: averageVelocity,
      trend,
      acceleration,
      movingAverages: { ma7, ma30 },
      volatility: this.calculateVolatility(aggregated),
      consistency: this.calculateConsistency(aggregated)
    }
  }
 
  detectAnomalies(data: TimeSeriesData[]): Anomaly[] {
    const anomalies: Anomaly[] = []
    
    // Calculate baseline statistics
    const mean = this.calculateMean(data)
    const stdDev = this.calculateStdDev(data, mean)
    
    // Z-score method for anomaly detection
    data.forEach((point, index) => {
      const zScore = Math.abs((point.value - mean) / stdDev)
      
      if (zScore > 3) {
        anomalies.push({
          timestamp: point.timestamp,
          value: point.value,
          zScore,
          type: point.value > mean ? 'spike' : 'drop',
          severity: this.calculateSeverity(zScore),
          context: this.getAnomalyContext(data, index)
        })
      }
    })
    
    // Pattern-based anomaly detection
    const patternAnomalies = this.detectPatternAnomalies(data)
    anomalies.push(...patternAnomalies)
    
    return anomalies
  }
 
  forecastTrend(
    historicalData: TimeSeriesData[],
    periods: number
  ): ForecastResult {
    // Apply multiple forecasting methods
    const methods = {
      linear: this.linearRegression(historicalData),
      exponential: this.exponentialSmoothing(historicalData),
      arima: this.arimaForecast(historicalData),
      prophet: this.prophetForecast(historicalData)
    }
    
    // Generate forecasts
    const forecasts = Object.entries(methods).map(([method, model]) => ({
      method,
      values: this.generateForecast(model, periods),
      confidence: this.calculateModelConfidence(model, historicalData)
    }))
    
    // Ensemble forecast
    const ensemble = this.ensembleForecast(forecasts)
    
    return {
      forecasts,
      ensemble,
      bestMethod: this.selectBestMethod(forecasts),
      confidenceIntervals: this.calculateConfidenceIntervals(ensemble),
      seasonality: this.detectSeasonality(historicalData)
    }
  }
 
  private detectSeasonality(data: TimeSeriesData[]): SeasonalityInfo {
    // Fourier transform for frequency analysis
    const frequencies = this.fourierTransform(data)
    
    // Identify dominant periods
    const dominantPeriods = this.identifyDominantPeriods(frequencies)
    
    // Calculate seasonal strength
    const seasonalStrength = this.calculateSeasonalStrength(
      data,
      dominantPeriods
    )
    
    return {
      hasSeasonality: seasonalStrength > 0.3,
      periods: dominantPeriods,
      strength: seasonalStrength,
      patterns: this.extractSeasonalPatterns(data, dominantPeriods)
    }
  }
}

// packages/analytics-engine/src/services/predictive-modeling.ts
import { MLModel } from '@mystoryflow/ml-core'
 
export class PredictiveModeler {
  private models: Map<string, MLModel>
 
  constructor() {
    this.models = new Map([
      ['commercial-success', new MLModel('xgboost', commercialSuccessConfig)],
      ['reader-engagement', new MLModel('neural-network', readerEngagementConfig)],
      ['viral-potential', new MLModel('random-forest', viralPotentialConfig)],
      ['quality-trajectory', new MLModel('lstm', qualityTrajectoryConfig)]
    ])
  }
 
  async predictCommercialSuccess(
    features: FeatureSet
  ): Promise<CommercialPrediction> {
    const model = this.models.get('commercial-success')!
    
    // Prepare features
    const preparedFeatures = this.prepareCommercialFeatures(features)
    
    // Make prediction
    const prediction = await model.predict(preparedFeatures)
    
    // Calculate feature importance
    const featureImportance = await model.getFeatureImportance()
    
    // Generate probability distribution
    const probabilities = await model.predictProbabilities(preparedFeatures)
 
    return {
      successProbability: prediction.probability,
      salesPotential: this.estimateSalesPotential(prediction, features),
      revenueProjection: this.projectRevenue(prediction, features),
      breakoutChance: probabilities.breakout,
      keyFactors: this.extractKeyFactors(featureImportance),
      marketSegments: await this.predictMarketSegments(features),
      timeline: this.generateCommercialTimeline(prediction)
    }
  }
 
  async predictReaderEngagement(
    features: FeatureSet
  ): Promise<EngagementPrediction> {
    const model = this.models.get('reader-engagement')!
    
    const prediction = await model.predict(
      this.prepareEngagementFeatures(features)
    )
 
    return {
      engagementScore: prediction.score,
      completionRate: prediction.components.completion,
      shareability: prediction.components.sharing,
      reviewLikelihood: prediction.components.reviews,
      rereadability: prediction.components.rereading,
      emotionalImpact: await this.predictEmotionalImpact(features),
      audienceSegments: await this.identifyAudienceSegments(features),
      viralElements: this.identifyViralElements(features, prediction)
    }
  }
 
  async simulateImprovements(
    currentFeatures: FeatureSet,
    improvements: ProposedImprovement[]
  ): Promise<ImprovementSimulation[]> {
    const simulations: ImprovementSimulation[] = []
 
    for (const improvement of improvements) {
      // Apply improvement to features
      const improvedFeatures = this.applyImprovement(
        currentFeatures,
        improvement
      )
      
      // Predict outcomes with improvement
      const improvedPredictions = {
        commercial: await this.predictCommercialSuccess(improvedFeatures),
        engagement: await this.predictReaderEngagement(improvedFeatures),
        quality: await this.predictQualityScore(improvedFeatures)
      }
      
      // Calculate impact
      const impact = this.calculateImprovementImpact(
        currentFeatures,
        improvedFeatures,
        improvedPredictions
      )
 
      simulations.push({
        improvement,
        predictedOutcomes: improvedPredictions,
        impact,
        effortVsReward: this.calculateROI(improvement.effort, impact),
        confidence: this.calculateSimulationConfidence(improvement)
      })
    }
 
    return simulations.sort((a, b) => b.effortVsReward - a.effortVsReward)
  }
}

// apps/story-analyzer/src/app/api/analytics/insights/route.ts
import { NextRequest, NextResponse } from 'next/server'
import { AdvancedAnalyticsEngine } from '@mystoryflow/analytics-engine'
import { withAuth } from '@mystoryflow/auth'
import { ChartGenerator } from '@mystoryflow/charts'
 
export async function POST(req: NextRequest) {
  try {
    const session = await withAuth(req)
    if (!session) {
      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
    }
 
    const {
      manuscriptId,
      timeRange,
      insightTypes = ['all'],
      visualizations = true
    } = await req.json()
 
    // Check user permissions for advanced analytics
    const hasAccess = await checkAnalyticsAccess(
      session.user.id,
      session.user.subscriptionTier
    )
    
    if (!hasAccess) {
      return NextResponse.json(
        { error: 'Advanced analytics requires Professional plan' },
        { status: 403 }
      )
    }
 
    // Generate insights
    const analyticsEngine = new AdvancedAnalyticsEngine()
    const insights = await analyticsEngine.generateComprehensiveInsights({
      userId: session.user.id,
      manuscriptId,
      timeRange: parseTimeRange(timeRange),
      insightTypes: insightTypes === ['all'] ? 
        ['writing-progress', 'quality-trends', 'market-positioning', 
         'predictive-success', 'improvement-velocity', 'engagement-patterns'] :
        insightTypes
    })
 
    // Generate visualizations if requested
    let charts = {}
    if (visualizations) {
      const chartGenerator = new ChartGenerator()
      charts = await chartGenerator.generateInsightCharts(insights)
    }
 
    return NextResponse.json({
      insights,
      charts,
      exportUrl: await generateExportUrl(insights, charts),
      generatedAt: new Date()
    })
  } catch (error) {
    console.error('Analytics generation failed:', error)
    return NextResponse.json(
      { error: 'Failed to generate insights' },
      { status: 500 }
    )
  }
}
 
// Real-time analytics streaming
export async function GET(req: NextRequest) {
  const session = await withAuth(req)
  if (!session) {
    return new Response('Unauthorized', { status: 401 })
  }
 
  const { searchParams } = new URL(req.url)
  const manuscriptId = searchParams.get('manuscriptId')
  const metrics = searchParams.get('metrics')?.split(',') || ['all']
 
  // Set up SSE stream
  const stream = new TransformStream()
  const writer = stream.writable.getWriter()
  const encoder = new TextEncoder()
 
  // Start streaming analytics
  const analyticsStream = new AnalyticsStreamService()
  
  analyticsStream.subscribe({
    userId: session.user.id,
    manuscriptId,
    metrics,
    onData: async (data) => {
      await writer.write(
        encoder.encode(`data: ${JSON.stringify(data)}\n\n`)
      )
    },
    onError: async (error) => {
      await writer.write(
        encoder.encode(`event: error\ndata: ${error.message}\n\n`)
      )
    }
  })
 
  return new Response(stream.readable, {
    headers: {
      'Content-Type': 'text/event-stream',
      'Cache-Control': 'no-cache',
      'Connection': 'keep-alive'
    }
  })
}

// packages/analytics-engine/src/services/comparative-analytics.ts
export class ComparativeAnalytics {
  async compareManuscripts(
    manuscriptIds: string[],
    dimensions: ComparisonDimension[]
  ): Promise<ComparativeAnalysis> {
    // Fetch data for all manuscripts
    const manuscriptData = await Promise.all(
      manuscriptIds.map(id => this.fetchManuscriptData(id))
    )
 
    // Perform dimension-specific comparisons
    const comparisons: DimensionComparison[] = []
    
    for (const dimension of dimensions) {
      const comparison = await this.compareDimension(
        manuscriptData,
        dimension
      )
      comparisons.push(comparison)
    }
 
    // Calculate relative strengths
    const relativeStrengths = this.calculateRelativeStrengths(
      manuscriptData,
      comparisons
    )
 
    // Identify patterns
    const patterns = await this.identifyComparativePatterns(
      manuscriptData,
      comparisons
    )
 
    // Generate insights
    const insights = await this.generateComparativeInsights(
      comparisons,
      patterns
    )
 
    return {
      manuscripts: manuscriptData.map(m => ({
        id: m.id,
        title: m.title,
        overallScore: m.scores.overall
      })),
      comparisons,
      relativeStrengths,
      patterns,
      insights,
      visualizations: {
        radar: this.generateRadarChartData(comparisons),
        timeSeries: this.generateTimeSeriesComparison(manuscriptData),
        heatmap: this.generateComparisonHeatmap(comparisons)
      }
    }
  }
 
  async compareToMarket(
    manuscriptId: string,
    comparisonGroup: 'genre' | 'all' | 'bestsellers'
  ): Promise<MarketComparison> {
    const manuscript = await this.fetchManuscriptData(manuscriptId)
    
    // Get comparison group data
    const groupData = await this.fetchComparisonGroup(
      comparisonGroup,
      manuscript.genre
    )
 
    // Calculate percentile ranks
    const percentiles = this.calculatePercentiles(manuscript, groupData)
 
    // Identify competitive advantages
    const advantages = this.identifyCompetitiveAdvantages(
      manuscript,
      groupData
    )
 
    // Find gaps and opportunities
    const gaps = this.identifyMarketGaps(manuscript, groupData)
 
    // Generate positioning strategy
    const positioning = await this.generatePositioningStrategy(
      manuscript,
      advantages,
      gaps
    )
 
    return {
      manuscript: {
        id: manuscript.id,
        title: manuscript.title,
        genre: manuscript.genre
      },
      comparisonGroup: {
        type: comparisonGroup,
        size: groupData.length,
        avgScore: this.calculateGroupAverage(groupData)
      },
      percentiles,
      advantages,
      gaps,
      positioning,
      recommendations: this.generateMarketRecommendations(
        percentiles,
        advantages,
        gaps
      )
    }
  }
}

-- Analytics data warehouse tables
CREATE TABLE analytics.manuscript_metrics (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  manuscript_id UUID REFERENCES analyzer.manuscripts(id),
  metric_date DATE,
  word_count INTEGER,
  quality_scores JSONB,
  engagement_metrics JSONB,
  writing_session_data JSONB,
  created_at TIMESTAMP DEFAULT NOW(),
  
  UNIQUE(manuscript_id, metric_date),
  INDEX idx_metrics_manuscript_date (manuscript_id, metric_date DESC)
);
 
-- Predictive model results
CREATE TABLE analytics.predictions (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  manuscript_id UUID REFERENCES analyzer.manuscripts(id),
  prediction_type VARCHAR(50),
  model_version VARCHAR(20),
  predictions JSONB,
  confidence_scores JSONB,
  feature_importance JSONB,
  created_at TIMESTAMP DEFAULT NOW(),
  
  INDEX idx_predictions_manuscript (manuscript_id),
  INDEX idx_predictions_type (prediction_type)
);
 
-- Time series data
CREATE TABLE analytics.time_series (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  entity_id UUID, -- Can be manuscript, user, or organization
  entity_type VARCHAR(50),
  metric_name VARCHAR(100),
  timestamp TIMESTAMP,
  value DECIMAL,
  metadata JSONB,
  
  INDEX idx_timeseries_entity (entity_id, entity_type, metric_name),
  INDEX idx_timeseries_time (timestamp DESC)
);
 
-- Comparative analysis results
CREATE TABLE analytics.comparative_analyses (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  analysis_type VARCHAR(50),
  entity_ids UUID[],
  comparison_data JSONB,
  insights JSONB,
  created_at TIMESTAMP DEFAULT NOW(),
  expires_at TIMESTAMP,
  
  INDEX idx_comparative_type (analysis_type),
  INDEX idx_comparative_entities (entity_ids)
);
 
-- Market intelligence data
CREATE TABLE analytics.market_intelligence (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  genre VARCHAR(50),
  period_start DATE,
  period_end DATE,
  trending_elements JSONB,
  market_metrics JSONB,
  competitive_landscape JSONB,
  
  UNIQUE(genre, period_start, period_end),
  INDEX idx_market_genre_period (genre, period_end DESC)
);
 
-- User analytics preferences
CREATE TABLE analytics.user_preferences (
  user_id UUID PRIMARY KEY REFERENCES auth.users(id),
  default_time_range VARCHAR(50),
  preferred_insights TEXT[],
  dashboard_layout JSONB,
  notification_settings JSONB,
  custom_metrics JSONB,
  
  INDEX idx_analytics_prefs_user (user_id)
);