AWS SDK Polly - Text-to-speech synthesis with streaming audio and SSML support

npm install @aws-sdk/client-polly

// lib/polly.ts
import { PollyClient } from "@aws-sdk/client-polly";
 
export const pollyClient = new PollyClient({
  region: process.env.AWS_REGION!,
  credentials: {
    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
  },
});

// app/api/speech/route.ts
import { SynthesizeSpeechCommand } from "@aws-sdk/client-polly";
import { pollyClient } from "@/lib/polly";
 
export async function POST(req: Request) {
  const { text, voiceId = "Joanna" } = await req.json();
 
  const command = new SynthesizeSpeechCommand({
    Text: text,
    OutputFormat: "mp3",
    VoiceId: voiceId,
    Engine: "neural",
  });
 
  const response = await pollyClient.send(command);
  const stream = response.AudioStream as ReadableStream;
 
  return new Response(stream, {
    headers: {
      "Content-Type": "audio/mpeg",
      "Cache-Control": "public, max-age=3600",
    },
  });
}

// app/components/TextToSpeech.tsx
"use client";
import { useState, useRef } from "react";
 
const VOICES = [
  { id: "Joanna", name: "Joanna (US English)", lang: "en-US" },
  { id: "Matthew", name: "Matthew (US English)", lang: "en-US" },
  { id: "Amy", name: "Amy (British English)", lang: "en-GB" },
  { id: "Lea", name: "Léa (French)", lang: "fr-FR" },
  { id: "Vicki", name: "Vicki (German)", lang: "de-DE" },
  { id: "Lucia", name: "Lucia (Spanish)", lang: "es-ES" },
];
 
export default function TextToSpeech() {
  const [text, setText] = useState("");
  const [voiceId, setVoiceId] = useState("Joanna");
  const [loading, setLoading] = useState(false);
  const audioRef = useRef<HTMLAudioElement>(null);
 
  async function handleSpeak() {
    if (!text.trim()) return;
    setLoading(true);
 
    try {
      const res = await fetch("/api/speech", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ text, voiceId }),
      });
 
      if (!res.ok) throw new Error("Speech synthesis failed");
 
      const blob = await res.blob();
      const url = URL.createObjectURL(blob);
 
      if (audioRef.current) {
        audioRef.current.src = url;
        audioRef.current.play();
      }
    } catch (error) {
      console.error("TTS error:", error);
    } finally {
      setLoading(false);
    }
  }
 
  return (
    <div className="max-w-md mx-auto p-6 space-y-4">
      <textarea
        value={text}
        onChange={(e) => setText(e.target.value)}
        placeholder="Enter text to speak..."
        rows={4}
        className="w-full border rounded px-3 py-2"
        maxLength={3000}
      />
      <div className="flex gap-3">
        <select
          value={voiceId}
          onChange={(e) => setVoiceId(e.target.value)}
          className="border rounded px-3 py-2"
        >
          {VOICES.map((v) => (
            <option key={v.id} value={v.id}>
              {v.name}
            </option>
          ))}
        </select>
        <button
          onClick={handleSpeak}
          disabled={loading || !text.trim()}
          className="bg-blue-600 text-white px-4 py-2 rounded disabled:opacity-50"
        >
          {loading ? "Generating..." : "Speak"}
        </button>
      </div>
      <audio ref={audioRef} controls className="w-full" />
      <p className="text-xs text-gray-500">{text.length}/3000 characters</p>
    </div>
  );
}

const command = new SynthesizeSpeechCommand({
  Text: `<speak>
    Welcome to our app.
    <break time="500ms"/>
    <prosody rate="slow" pitch="+10%">
      This part is spoken slowly with a higher pitch.
    </prosody>
    <emphasis level="strong">This is important.</emphasis>
    <say-as interpret-as="date" format="mdy">12/25/2025</say-as>
  </speak>`,
  TextType: "ssml",
  OutputFormat: "mp3",
  VoiceId: "Joanna",
  Engine: "neural",
});

import { DescribeVoicesCommand } from "@aws-sdk/client-polly";
 
export async function GET() {
  const command = new DescribeVoicesCommand({
    Engine: "neural",
    LanguageCode: "en-US",
  });
 
  const response = await pollyClient.send(command);
  const voices = response.Voices?.map((v) => ({
    id: v.Id,
    name: v.Name,
    gender: v.Gender,
    languageName: v.LanguageName,
  }));
 
  return Response.json(voices);
}

const command = new SynthesizeSpeechCommand({
  Text: "Hello, how are you today?",
  OutputFormat: "json",
  VoiceId: "Joanna",
  Engine: "neural",
  SpeechMarkTypes: ["word", "sentence"],
});
 
const response = await pollyClient.send(command);
// Returns JSONL with timing data for each word/sentence
// {"time":0,"type":"sentence","start":0,"end":25,"value":"Hello, how are you today?"}
// {"time":0,"type":"word","start":0,"end":5,"value":"Hello"}

"use server";
import { SynthesizeSpeechCommand } from "@aws-sdk/client-polly";
import { pollyClient } from "@/lib/polly";
 
export async function synthesizeSpeech(text: string, voiceId: string = "Joanna") {
  const command = new SynthesizeSpeechCommand({
    Text: text.slice(0, 3000),
    OutputFormat: "mp3",
    VoiceId: voiceId,
    Engine: "neural",
  });
 
  const response = await pollyClient.send(command);
  const chunks: Uint8Array[] = [];
  const stream = response.AudioStream as AsyncIterable<Uint8Array>;
 
  for await (const chunk of stream) {
    chunks.push(chunk);
  }
 
  const buffer = Buffer.concat(chunks);
  return buffer.toString("base64");
}

import type {
  SynthesizeSpeechCommandInput,
  VoiceId,
  Engine,
} from "@aws-sdk/client-polly";
 
const voiceId: VoiceId = "Joanna";
const engine: Engine = "neural";
 
const input: SynthesizeSpeechCommandInput = {
  Text: "Hello",
  OutputFormat: "mp3",
  VoiceId: voiceId,
  Engine: engine,
};