F030 - Speed & Scalability Optimization

// packages/manuscript-analysis/src/services/optimized-analysis-engine.ts
import { Readable } from 'stream'
import pLimit from 'p-limit'
import { LRUCache } from 'lru-cache'
 
export class OptimizedAnalysisEngine {
  private chunkLimit = pLimit(5) // Process 5 chunks in parallel
  private cache = new LRUCache<string, any>({
    max: 100,
    ttl: 1000 * 60 * 60 // 1 hour cache
  })
 
  async analyzeManuscriptStream(
    manuscriptId: string,
    content: string,
    onProgress: (progress: number) => void
  ): Promise<Readable> {
    // Check cache first
    const cacheKey = this.getCacheKey(manuscriptId, content)
    const cached = this.cache.get(cacheKey)
    if (cached) {
      return Readable.from([cached])
    }
 
    // Split into optimal chunks for parallel processing
    const chunks = this.createOptimalChunks(content)
    const totalChunks = chunks.length
    let processedChunks = 0
 
    // Create readable stream for real-time updates
    const stream = new Readable({
      read() {}
    })
 
    // Process chunks in parallel with limit
    const chunkPromises = chunks.map((chunk, index) => 
      this.chunkLimit(async () => {
        const result = await this.analyzeChunk(chunk, index)
        processedChunks++
        
        // Update progress
        const progress = Math.round((processedChunks / totalChunks) * 100)
        onProgress(progress)
        
        // Stream partial results
        stream.push(JSON.stringify({
          type: 'partial',
          chunkIndex: index,
          data: result
        }) + '\n')
        
        return result
      })
    )
 
    // Aggregate results
    Promise.all(chunkPromises).then(async (results) => {
      const aggregated = await this.aggregateResults(results)
      
      // Cache the result
      this.cache.set(cacheKey, aggregated)
      
      // Send final result
      stream.push(JSON.stringify({
        type: 'complete',
        data: aggregated
      }) + '\n')
      
      stream.push(null) // End stream
    }).catch(error => {
      stream.destroy(error)
    })
 
    return stream
  }
 
  private createOptimalChunks(content: string): string[] {
    const MAX_TOKENS_PER_CHUNK = 3000
    const OVERLAP_TOKENS = 200
    
    // Use GPT tokenizer for accurate counting
    const tokens = this.tokenize(content)
    const chunks: string[] = []
    
    for (let i = 0; i < tokens.length; i += MAX_TOKENS_PER_CHUNK - OVERLAP_TOKENS) {
      const chunkTokens = tokens.slice(i, i + MAX_TOKENS_PER_CHUNK)
      chunks.push(this.detokenize(chunkTokens))
    }
    
    return chunks
  }
 
  private async analyzeChunk(chunk: string, index: number): Promise<any> {
    // Use lighter model for initial chunks, full model for critical sections
    const model = index < 3 ? 'gpt-3.5-turbo' : 'gpt-4'
    
    const response = await fetch('https://api.openai.com/v1/chat/completions', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        model,
        messages: [{
          role: 'system',
          content: 'Analyze this manuscript section...'
        }, {
          role: 'user',
          content: chunk
        }],
        temperature: 0.3,
        max_tokens: 1000,
        stream: false
      })
    })
 
    return response.json()
  }
 
  private async aggregateResults(results: any[]): Promise<any> {
    // Intelligent aggregation of chunk analyses
    const aggregated = {
      overall_score: 0,
      category_scores: {},
      strengths: [],
      weaknesses: [],
      detailed_feedback: {}
    }
 
    // Weight early chapters more heavily
    const weights = results.map((_, i) => Math.max(1, 5 - i * 0.5))
    const totalWeight = weights.reduce((a, b) => a + b, 0)
 
    results.forEach((result, index) => {
      const weight = weights[index] / totalWeight
      // Aggregate scores with weighting
      aggregated.overall_score += result.score * weight
      // Merge other fields intelligently
    })
 
    return aggregated
  }
}

// packages/database/src/optimizations.ts
import { createClient } from '@mystoryflow/database/server'
 
// Materialized view for dashboard stats
export const createMaterializedViews = async () => {
  const supabase = getSupabaseBrowserClient()
  
  // User dashboard stats
  await supabase.rpc('create_materialized_view', {
    view_name: 'user_dashboard_stats',
    query: `
      SELECT 
        u.id as user_id,
        COUNT(DISTINCT m.id) as total_manuscripts,
        COUNT(DISTINCT ma.id) as total_analyses,
        AVG(ma.overall_score) as average_score,
        MAX(ma.overall_score) as best_score,
        COUNT(DISTINCT cs.id) as total_consultations,
        COALESCE(SUM(m.word_count), 0) as total_words_analyzed
      FROM users u
      LEFT JOIN manuscripts m ON m.user_id = u.id
      LEFT JOIN manuscript_analyses ma ON ma.manuscript_id = m.id
      LEFT JOIN consultation_sessions cs ON cs.author_id = u.id
      GROUP BY u.id
    `,
    refresh_interval: '1 hour'
  })
 
  // Genre performance stats
  await supabase.rpc('create_materialized_view', {
    view_name: 'genre_performance_stats',
    query: `
      SELECT
        gd.primary_genre as genre,
        COUNT(DISTINCT m.id) as manuscript_count,
        AVG(ma.overall_score) as avg_score,
        PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ma.overall_score) as median_score,
        AVG(ma.processing_time_ms) as avg_processing_time
      FROM manuscripts m
      JOIN genre_detections gd ON gd.manuscript_id = m.id
      JOIN manuscript_analyses ma ON ma.manuscript_id = m.id
      GROUP BY gd.primary_genre
    `,
    refresh_interval: '6 hours'
  })
}
 
// Optimized queries with proper indexing
export const getOptimizedUserDashboard = async (userId: string) => {
  const supabase = getSupabaseBrowserClient()
  
  // Use materialized view for instant results
  const { data: stats } = await supabase
    .from('analyzer.user_dashboard_stats')
    .select('*')
    .eq('user_id', userId)
    .single()
 
  // Get recent items with optimized query
  const { data: recentItems } = await supabase
    .from('analyzer.manuscripts')
    .select(`
      id,
      title,
      created_at,
      manuscript_analyses!inner (
        overall_score,
        created_at
      )
    `)
    .eq('user_id', userId)
    .order('manuscript_analyses.created_at', { ascending: false })
    .limit(5)
 
  return { stats, recentItems }
}

// next.config.js
module.exports = {
  experimental: {
    optimizeCss: true,
    optimizePackageImports: ['@mystoryflow/ui', 'lucide-react']
  },
  images: {
    domains: ['cdn.mystoryflow.com'],
    formats: ['image/avif', 'image/webp']
  },
  compress: true,
  poweredByHeader: false,
  
  // Split chunks for better caching
  webpack: (config, { isServer }) => {
    if (!isServer) {
      config.optimization.splitChunks = {
        chunks: 'all',
        cacheGroups: {
          default: false,
          vendors: false,
          framework: {
            name: 'framework',
            chunks: 'all',
            test: /[\\/]node_modules[\\/](react|react-dom|next)[\\/]/,
            priority: 40,
            enforce: true
          },
          lib: {
            test: /[\\/]node_modules[\\/]/,
            name(module) {
              const packageName = module.context.match(
                /[\\/]node_modules[\\/](.*?)([[\\/]|$)/
              )[1]
              return `npm.${packageName.replace('@', '')}`
            },
            priority: 30,
            minChunks: 1,
            reuseExistingChunk: true
          },
          commons: {
            name: 'commons',
            minChunks: 2,
            priority: 20
          },
          shared: {
            name(module, chunks) {
              return crypto
                .createHash('sha1')
                .update(chunks.reduce((acc, chunk) => acc + chunk.name, ''))
                .digest('hex') + (isServer ? '-server' : '-client')
            },
            priority: 10,
            minChunks: 2,
            reuseExistingChunk: true
          }
        }
      }
    }
    return config
  }
}

// apps/analyzer-app/src/app/api/analysis/status/route.ts
import { NextRequest } from 'next/server'
 
export const runtime = 'edge'
 
// Use KV store for ultra-fast status checks
const statusCache = new Map<string, any>()
 
export async function GET(request: NextRequest) {
  const manuscriptId = request.nextUrl.searchParams.get('manuscriptId')
  
  if (!manuscriptId) {
    return new Response('Missing manuscriptId', { status: 400 })
  }
 
  // Check memory cache first (edge runtime persists between requests)
  const cached = statusCache.get(manuscriptId)
  if (cached && Date.now() - cached.timestamp < 5000) {
    return new Response(JSON.stringify(cached.data), {
      headers: {
        'Content-Type': 'application/json',
        'Cache-Control': 'public, max-age=5'
      }
    })
  }
 
  // Fetch from database with connection pooling
  const response = await fetch(
    `${process.env.SUPABASE_URL}/rest/v1/manuscript_analyses?manuscript_id=eq.${manuscriptId}&order=created_at.desc&limit=1`,
    {
      headers: {
        'apikey': process.env.SUPABASE_ANON_KEY!,
        'Authorization': `Bearer ${process.env.SUPABASE_ANON_KEY}`
      }
    }
  )
 
  const data = await response.json()
  const analysis = data[0]
 
  const status = {
    status: analysis ? 'completed' : 'processing',
    progress: analysis ? 100 : 50,
    completedAt: analysis?.created_at
  }
 
  // Update cache
  statusCache.set(manuscriptId, {
    data: status,
    timestamp: Date.now()
  })
 
  return new Response(JSON.stringify(status), {
    headers: {
      'Content-Type': 'application/json',
      'Cache-Control': 'public, max-age=5'
    }
  })
}

// apps/analyzer-app/src/hooks/useOptimizedData.ts
import { useQuery } from '@tanstack/react-query'
import { useState, useEffect, useRef } from 'react'
 
// Virtual scrolling for large lists
export function useVirtualList<T>({
  items,
  height,
  itemHeight,
  overscan = 5
}: {
  items: T[]
  height: number
  itemHeight: number
  overscan?: number
}) {
  const [scrollTop, setScrollTop] = useState(0)
  const scrollElementRef = useRef<HTMLDivElement>(null)
 
  const startIndex = Math.max(0, Math.floor(scrollTop / itemHeight) - overscan)
  const endIndex = Math.min(
    items.length - 1,
    Math.ceil((scrollTop + height) / itemHeight) + overscan
  )
 
  const visibleItems = items.slice(startIndex, endIndex + 1)
  const totalHeight = items.length * itemHeight
  const offsetY = startIndex * itemHeight
 
  useEffect(() => {
    const handleScroll = () => {
      if (scrollElementRef.current) {
        setScrollTop(scrollElementRef.current.scrollTop)
      }
    }
 
    const element = scrollElementRef.current
    element?.addEventListener('scroll', handleScroll, { passive: true })
    
    return () => {
      element?.removeEventListener('scroll', handleScroll)
    }
  }, [])
 
  return {
    scrollElementRef,
    visibleItems,
    totalHeight,
    offsetY
  }
}
 
// Optimized data fetching with prefetching
export function useManuscriptAnalysis(manuscriptId: string) {
  const queryClient = useQueryClient()
  
  // Main query
  const analysisQuery = useQuery({
    queryKey: ['analysis', manuscriptId],
    queryFn: () => fetchAnalysis(manuscriptId),
    staleTime: 1000 * 60 * 5, // 5 minutes
    cacheTime: 1000 * 60 * 30 // 30 minutes
  })
 
  // Prefetch related data
  useEffect(() => {
    if (analysisQuery.data) {
      // Prefetch author's other manuscripts
      queryClient.prefetchQuery({
        queryKey: ['manuscripts', analysisQuery.data.userId],
        queryFn: () => fetchUserManuscripts(analysisQuery.data.userId)
      })
 
      // Prefetch similar genre analyses
      queryClient.prefetchQuery({
        queryKey: ['genre-comparisons', analysisQuery.data.genre],
        queryFn: () => fetchGenreComparisons(analysisQuery.data.genre)
      })
    }
  }, [analysisQuery.data, queryClient])
 
  return analysisQuery
}

// apps/analyzer-app/src/lib/image-optimization.ts
import { getPlaiceholder } from 'plaiceholder'
 
export async function getOptimizedImage(src: string) {
  try {
    const { base64, img } = await getPlaiceholder(src)
    
    return {
      ...img,
      blurDataURL: base64
    }
  } catch (error) {
    return {
      src,
      height: 600,
      width: 800
    }
  }
}
 
// Usage in component
export async function CoachProfile({ coach }: { coach: any }) {
  const optimizedAvatar = await getOptimizedImage(coach.avatar_url)
  
  return (
    <div>
      <Image
        {...optimizedAvatar}
        alt={coach.full_name}
        placeholder="blur"
        loading="lazy"
        sizes="(max-width: 768px) 100vw, (max-width: 1200px) 50vw, 33vw"
      />
    </div>
  )
}

// public/sw.js
const CACHE_NAME = 'story-analyzer-v1'
const urlsToCache = [
  '/',
  '/offline',
  '/manifest.json',
  '/_next/static/css/app.css',
  '/_next/static/js/app.js'
]
 
self.addEventListener('install', (event) => {
  event.waitUntil(
    caches.open(CACHE_NAME).then((cache) => {
      return cache.addAll(urlsToCache)
    })
  )
})
 
self.addEventListener('fetch', (event) => {
  // Cache-first strategy for static assets
  if (event.request.url.includes('/_next/static/')) {
    event.respondWith(
      caches.match(event.request).then((response) => {
        return response || fetch(event.request).then((response) => {
          return caches.open(CACHE_NAME).then((cache) => {
            cache.put(event.request, response.clone())
            return response
          })
        })
      })
    )
    return
  }
 
  // Network-first strategy for API calls
  if (event.request.url.includes('/api/')) {
    event.respondWith(
      fetch(event.request)
        .then((response) => {
          const responseClone = response.clone()
          caches.open(CACHE_NAME).then((cache) => {
            cache.put(event.request, responseClone)
          })
          return response
        })
        .catch(() => {
          return caches.match(event.request)
        })
    )
    return
  }
 
  // Default strategy
  event.respondWith(
    caches.match(event.request).then((response) => {
      return response || fetch(event.request)
    })
  )
})

// apps/analyzer-app/src/lib/performance-monitor.ts
export class PerformanceMonitor {
  private metrics: Map<string, number[]> = new Map()
 
  measureApiCall(endpoint: string, duration: number) {
    if (!this.metrics.has(endpoint)) {
      this.metrics.set(endpoint, [])
    }
    
    const calls = this.metrics.get(endpoint)!
    calls.push(duration)
    
    // Keep only last 100 calls
    if (calls.length > 100) {
      calls.shift()
    }
    
    // Alert if performance degrades
    const avg = calls.reduce((a, b) => a + b) / calls.length
    if (avg > 1000) { // 1 second threshold
      this.alertSlowEndpoint(endpoint, avg)
    }
  }
 
  measureComponentRender(componentName: string, duration: number) {
    if (duration > 16) { // 60fps threshold
      console.warn(`Slow render: ${componentName} took ${duration}ms`)
    }
  }
 
  private alertSlowEndpoint(endpoint: string, avgDuration: number) {
    // Send to monitoring service
    fetch('/api/monitoring/performance', {
      method: 'POST',
      body: JSON.stringify({
        type: 'slow_endpoint',
        endpoint,
        averageDuration: avgDuration,
        timestamp: new Date().toISOString()
      })
    })
  }
}
 
// Usage
const monitor = new PerformanceMonitor()
 
export async function fetchWithMonitoring(url: string, options?: RequestInit) {
  const start = performance.now()
  
  try {
    const response = await fetch(url, options)
    const duration = performance.now() - start
    
    monitor.measureApiCall(url, duration)
    
    return response
  } catch (error) {
    const duration = performance.now() - start
    monitor.measureApiCall(url, duration)
    throw error
  }
}