001/** 002 * Copyright 2001 Sun Microsystems, Inc. 003 * 004 * See the file "license.terms" for information on usage and 005 * redistribution of this file, and for a DISCLAIMER OF ALL 006 * WARRANTIES. 007 */ 008package com.sun.speech.freetts.audio; 009 010import javax.sound.sampled.AudioFormat; 011import javax.sound.sampled.AudioSystem; 012import javax.sound.sampled.DataLine; 013import javax.sound.sampled.FloatControl; 014import javax.sound.sampled.LineEvent; 015import javax.sound.sampled.LineListener; 016import javax.sound.sampled.LineUnavailableException; 017import javax.sound.sampled.SourceDataLine; 018 019import com.sun.speech.freetts.util.BulkTimer; 020import com.sun.speech.freetts.util.Timer; 021import com.sun.speech.freetts.util.Utilities; 022 023/** 024 * Streams audio to java audio. This class provides a low latency 025 * method of sending audio output through the javax.sound audio API. 026 * Audio data is sent in small sets to the audio system allowing it to 027 * be played soon after it is generated. 028 * 029 * Unfortunately, the current release of the JDK (JDK 1.4 beta 2) has 030 * a bug or two in 031 * the implementation of 'SourceDataLine.drain'. A workaround solution that 032 * sleep/waits on SourceDataLine.isActive is used here instead. To 033 * disable the work around (i.e use the real 'drain') set the 034 * property: 035 * <p> 036 * <code> 037 * com.sun.speech.freetts.audio.AudioPlayer.drainWorksProperly; 038 * </code> 039 * to <code>true</code>. 040 * 041 * If the workaround is enabled, the line.isActive method will be 042 * performed periodically. The period of the test can be controlled 043 * with: 044 * 045 * <p> 046 * <code> 047 * com.sun.speech.freetts.audio.AudioPlayer.drainDelay" 048 * </code> 049 * 050 * <p> 051 * The default if 5ms. 052 * 053 * <p> 054 * The property 055 * <code> 056 * com.sun.speech.freetts.audio.AudioPlayer.bufferSize" 057 * </code> 058 * 059 * <p> 060 * Controls the audio buffer size, it defaults to 8192 061 * 062 * <p> 063 * Even with this drain work around, there are some issues with this 064 * class. The workaround drain is not completely reliable. 065 * A <code>resume</code> following a <code>pause</code> does not 066 * always continue at the proper position in the audio. On a rare 067 * occasion, sound output will be repeated a number of times. This may 068 * be related to bug 4421330 in the Bug Parade database. 069 * 070 * 071 */ 072public class JavaStreamingAudioPlayer implements AudioPlayer { 073 074 private volatile boolean paused; 075 private volatile boolean done = false; 076 private volatile boolean cancelled = false; 077 078 private SourceDataLine line; 079 private float volume = 1.0f; // the current volume 080 private long timeOffset = 0L; 081 private BulkTimer timer = new BulkTimer(); 082 083 // default format is 8khz 084 private AudioFormat defaultFormat = 085 new AudioFormat(8000f, 16, 1, true, true); 086 private AudioFormat currentFormat = defaultFormat; 087 088 private boolean debug = false; 089 private boolean audioMetrics = false; 090 private boolean firstSample = true; 091 092 private long cancelDelay; 093 private long drainDelay; 094 private long openFailDelayMs; 095 private long totalOpenFailDelayMs; 096 097 private Object openLock = new Object(); 098 private Object lineLock = new Object(); 099 100 101 /** 102 * controls the buffering to java audio 103 */ 104 private final static int AUDIO_BUFFER_SIZE = Utilities.getInteger( 105 "com.sun.speech.freetts.audio.AudioPlayer.bufferSize", 8192).intValue(); 106 107 /** 108 * controls the number of bytes of audio to write to the buffer 109 * for each call to write() 110 */ 111 private final static int BYTES_PER_WRITE = Utilities.getInteger 112 ("com.sun.speech.freetts.audio.AudioPlayer.bytesPerWrite", 160).intValue(); 113 114 115 /** 116 * Constructs a default JavaStreamingAudioPlayer 117 */ 118 public JavaStreamingAudioPlayer() { 119 debug = Utilities.getBoolean 120 ("com.sun.speech.freetts.audio.AudioPlayer.debug"); 121 cancelDelay = Utilities.getLong 122 ("com.sun.speech.freetts.audio.AudioPlayer.cancelDelay", 123 0L).longValue(); 124 drainDelay = Utilities.getLong 125 ("com.sun.speech.freetts.audio.AudioPlayer.drainDelay", 126 150L).longValue(); 127 openFailDelayMs = Utilities.getLong 128 ("com.sun.speech.freetts.audio.AudioPlayer.openFailDelayMs", 129 0L).longValue(); 130 totalOpenFailDelayMs = Utilities.getLong 131 ("com.sun.speech.freetts.audio.AudioPlayer.totalOpenFailDelayMs", 132 0L).longValue(); 133 audioMetrics = Utilities.getBoolean 134 ("com.sun.speech.freetts.audio.AudioPlayer.showAudioMetrics"); 135 136 line = null; 137 setPaused(false); 138 } 139 140 /** 141 * Sets the audio format for this player 142 * 143 * @param format the audio format 144 * 145 * @throws UnsupportedOperationException if the line cannot be opened with 146 * the given format 147 */ 148 public synchronized void setAudioFormat(AudioFormat format) { 149 currentFormat = format; 150 debugPrint("AF changed to " + format); 151 } 152 153 154 /** 155 * Gets the audio format for this player 156 * 157 * @return format the audio format 158 */ 159 public AudioFormat getAudioFormat() { 160 return currentFormat; 161 } 162 163 /** 164 * Starts the first sample timer 165 */ 166 public void startFirstSampleTimer() { 167 timer.start("firstAudio"); 168 firstSample = true; 169 } 170 171 172 /** 173 * Opens the audio 174 * 175 * @param format the format for the audio 176 * 177 * @throws UnsupportedOperationException if the line cannot be opened with 178 * the given format 179 */ 180 private synchronized void openLine(AudioFormat format) { 181 synchronized (lineLock) { 182 if (line != null) { 183 line.close(); 184 line = null; 185 } 186 } 187 DataLine.Info info = new DataLine.Info(SourceDataLine.class, format); 188 189 boolean opened = false; 190 long totalDelayMs = 0; 191 192 do { 193 try { 194 line = (SourceDataLine) AudioSystem.getLine(info); 195 line.addLineListener(new JavaStreamLineListener()); 196 197 synchronized (openLock) { 198 line.open(format, AUDIO_BUFFER_SIZE); 199 try { 200 openLock.wait(); 201 } catch (InterruptedException ie) { 202 ie.printStackTrace(); 203 } 204 opened = true; 205 } 206 } catch (LineUnavailableException lue) { 207 System.err.println("LINE UNAVAILABLE: " + 208 "Format is " + currentFormat); 209 try { 210 Thread.sleep(openFailDelayMs); 211 totalDelayMs += openFailDelayMs; 212 } catch (InterruptedException ie) { 213 ie.printStackTrace(); 214 } 215 } 216 } while (!opened && totalDelayMs < totalOpenFailDelayMs); 217 218 if (opened) { 219 setVolume(line, volume); 220 resetTime(); 221 if (isPaused() && line.isRunning()) { 222 line.stop(); 223 } else { 224 line.start(); 225 } 226 } else { 227 if (line != null) { 228 line.close(); 229 } 230 line = null; 231 } 232 } 233 234 235 /** 236 * Pauses audio output 237 */ 238 public synchronized void pause() { 239 if (!isPaused()) { 240 setPaused(true); 241 if (line != null) { 242 line.stop(); 243 } 244 } 245 } 246 247 /** 248 * Resumes audio output 249 */ 250 public synchronized void resume() { 251 if (isPaused()) { 252 setPaused(false); 253 if (!isCancelled() && line != null) { 254 line.start(); 255 notify(); 256 } 257 } 258 } 259 260 261 /** 262 * Cancels currently playing audio. 263 */ 264 265 // [[[ WORKAROUND TODO 266 // The "Thread.sleep(cancelDelay)" is added to fix a problem in the 267 // FreeTTSEmacspeak demo. The problem was that the engine would 268 // stutter after using it for a while. Adding this sleep() fixed the 269 // problem. If we later find out that this problem no longer exists, 270 // we should remove the thread.sleep(). ]]] 271 public void cancel() { 272 debugPrint("cancelling..."); 273 274 if (audioMetrics) { 275 timer.start("audioCancel"); 276 } 277 278 if (cancelDelay > 0) { 279 try { 280 Thread.sleep(cancelDelay); 281 } catch (InterruptedException ie) { 282 ie.printStackTrace(); 283 } 284 } 285 286 synchronized (lineLock) { 287 if (line != null && line.isRunning()) { 288 line.stop(); 289 line.flush(); 290 } 291 } 292 293 /* sets 'cancelled' to false, which breaks the write while loop */ 294 synchronized (this) { 295 cancelled = true; 296 notify(); 297 } 298 299 if (audioMetrics) { 300 timer.stop("audioCancel"); 301 Timer.showTimesShortTitle(""); 302 timer.getTimer("audioCancel").showTimesShort(0); 303 } 304 305 debugPrint("...cancelled"); 306 } 307 308 /** 309 * Prepares for another batch of output. Larger groups of output 310 * (such as all output associated with a single FreeTTSSpeakable) 311 * should be grouped between a reset/drain pair. 312 */ 313 public synchronized void reset() { 314 timer.start("audioOut"); 315 if (line != null) { 316 waitResume(); 317 if (isCancelled() && !isDone()) { 318 cancelled = false; 319 line.start(); 320 } 321 } 322 } 323 324 /** 325 * Closes this audio player 326 */ 327 public synchronized void close() { 328 done = true; 329 if (line != null && line.isOpen()) { 330 line.close(); 331 line = null; 332 notify(); 333 } 334 } 335 336 337 /** 338 * Returns the current volume. 339 * 340 * @return the current volume (between 0 and 1) 341 */ 342 public float getVolume() { 343 return volume; 344 } 345 346 /** 347 * Sets the current volume. 348 * 349 * @param volume the current volume (between 0 and 1) 350 */ 351 public void setVolume(float volume) { 352 if (volume > 1.0f) { 353 volume = 1.0f; 354 } 355 if (volume < 0.0f) { 356 volume = 0.0f; 357 } 358 this.volume = volume; 359 } 360 361 /** 362 * Sets us in pause mode 363 * 364 * @param state true if we are paused 365 */ 366 private void setPaused(boolean state) { 367 paused = state; 368 } 369 370 /** 371 * Returns true if we are in pause mode 372 * 373 * @return true if paused 374 */ 375 private boolean isPaused() { 376 return paused; 377 } 378 379 /** 380 * Sets the volume on the given clip 381 * 382 * @param line the line to set the volume on 383 * @param vol the volume (range 0 to 1) 384 */ 385 private void setVolume(SourceDataLine line, float vol) { 386 if (line != null && 387 line.isControlSupported (FloatControl.Type.MASTER_GAIN)) { 388 FloatControl volumeControl = 389 (FloatControl) line.getControl (FloatControl.Type.MASTER_GAIN); 390 float range = volumeControl.getMaximum() - 391 volumeControl.getMinimum(); 392 volumeControl.setValue(vol * range + volumeControl.getMinimum()); 393 } 394 } 395 396 /** 397 * Starts the output of a set of data. 398 * For this JavaStreamingAudioPlayer, it actually opens the audio line. 399 * Since this is a streaming audio player, the <code>size</code> 400 * parameter has no meaning and effect at all, so any value can be used. 401 * Audio data for a single utterance should be grouped 402 * between begin/end pairs. 403 * 404 * @param size supposedly the size of data between now and the end, 405 * but since this is a streaming audio player, this parameter 406 * has no meaning and effect at all 407 */ 408 public void begin(int size) { 409 debugPrint("opening Stream..."); 410 openLine(currentFormat); 411 reset(); 412 debugPrint("...Stream opened"); 413 } 414 415 /** 416 * Marks the end of a set of data. Audio data for a single 417 * utterance should be groupd between begin/end pairs. 418 * 419 * @return true if the audio was output properly, false if the 420 * output was cancelled or interrupted. 421 * 422 */ 423 public synchronized boolean end() { 424 if (line != null) { 425 drain(); 426 synchronized (lineLock) { 427 line.close(); 428 line = null; 429 } 430 notify(); 431 debugPrint("ended stream..."); 432 } 433 return true; 434 } 435 436 /** 437 * Waits for all queued audio to be played 438 * 439 * @return true if the audio played to completion, false if 440 * the audio was stopped 441 * 442 * [[[ WORKAROUND TODO 443 * The javax.sound.sampled drain is almost working properly. On 444 * linux, there is still a little bit of sound that needs to go 445 * out, even after drain is called. Thus, the drainDelay. We 446 * wait for a few hundred milliseconds while the data is really 447 * drained out of the system 448 * ]]] 449 */ 450 public boolean drain() { 451 if (line != null) { 452 debugPrint("started draining..."); 453 if (line.isOpen()) { 454 line.drain(); 455 if (drainDelay > 0L) { 456 try { 457 Thread.sleep(drainDelay); 458 } catch (InterruptedException ie) { 459 } 460 } 461 } 462 debugPrint("...finished draining"); 463 } 464 timer.stop("audioOut"); 465 466 return !isCancelled(); 467 } 468 469 /** 470 * Gets the amount of played since the last mark 471 * 472 * @return the amount of audio in milliseconds 473 */ 474 public synchronized long getTime() { 475 return (line.getMicrosecondPosition() - timeOffset) / 1000L; 476 } 477 478 479 /** 480 * Resets the audio clock 481 */ 482 public synchronized void resetTime() { 483 timeOffset = line.getMicrosecondPosition(); 484 } 485 486 487 488 /** 489 * Writes the given bytes to the audio stream 490 * 491 * @param audioData audio data to write to the device 492 * 493 * @return <code>true</code> of the write completed successfully, 494 * <code> false </code>if the write was cancelled. 495 */ 496 public boolean write(byte[] audioData) { 497 return write(audioData, 0, audioData.length); 498 } 499 500 /** 501 * Writes the given bytes to the audio stream 502 * 503 * @param bytes audio data to write to the device 504 * @param offset the offset into the buffer 505 * @param size the size into the buffer 506 * 507 * @return <code>true</code> of the write completed successfully, 508 * <code> false </code>if the write was cancelled. 509 */ 510 public boolean write(byte[] bytes, int offset, int size) { 511 if (line == null) { 512 return false; 513 } 514 515 int bytesRemaining = size; 516 int curIndex = offset; 517 518 if (firstSample) { 519 firstSample = false; 520 timer.stop("firstAudio"); 521 if (audioMetrics) { 522 Timer.showTimesShortTitle(""); 523 timer.getTimer("firstAudio").showTimesShort(0); 524 } 525 } 526 debugPrint(" au write " + bytesRemaining + 527 " pos " + line.getMicrosecondPosition() 528 + " avail " + line.available() + " bsz " + 529 line.getBufferSize()); 530 531 while (bytesRemaining > 0 && !isCancelled()) { 532 533 if (!waitResume()) { 534 return false; 535 } 536 537 debugPrint(" queueing cur " + curIndex + " br " + bytesRemaining); 538 int bytesWritten; 539 540 synchronized (lineLock) { 541 bytesWritten = line.write 542 (bytes, curIndex, 543 Math.min(BYTES_PER_WRITE, bytesRemaining)); 544 545 if (bytesWritten != bytesRemaining) { 546 debugPrint 547 ("RETRY! bw" +bytesWritten + " br " + bytesRemaining); 548 } 549 // System.out.println("BytesWritten: " + bytesWritten); 550 curIndex += bytesWritten; 551 bytesRemaining -= bytesWritten; 552 } 553 554 debugPrint(" wrote " + " cur " + curIndex 555 + " br " + bytesRemaining 556 + " bw " + bytesWritten); 557 558 } 559 return !isCancelled() && !isDone(); 560 } 561 562 563 /** 564 * Waits for resume. If this audio player 565 * is paused waits for the player to be resumed. 566 * Returns if resumed, cancelled or shutdown. 567 * 568 * @return true if the output has been resumed, false if the 569 * output has been cancelled or shutdown. 570 */ 571 private synchronized boolean waitResume() { 572 while (isPaused() && !isCancelled() && !isDone()) { 573 try { 574 debugPrint(" paused waiting "); 575 wait(); 576 } catch (InterruptedException ie) { 577 } 578 } 579 580 return !isCancelled() && !isDone(); 581 } 582 583 584 /** 585 * Returns the name of this audioplayer 586 * 587 * @return the name of the audio player 588 */ 589 public String toString() { 590 return "JavaStreamingAudioPlayer"; 591 } 592 593 594 /** 595 * Outputs a debug message if debugging is turned on 596 * 597 * @param msg the message to output 598 */ 599 private void debugPrint(String msg) { 600 if (debug) { 601 System.out.println(toString() + ": " + msg); 602 } 603 } 604 605 /** 606 * Shows metrics for this audio player 607 */ 608 public void showMetrics() { 609 timer.show("JavaStreamingAudioPlayer"); 610 } 611 612 /** 613 * Determines if the output has been canceled. Access to the 614 * canceled variable should be within a synchronized block such 615 * as this to ensure that access is coherent. 616 * 617 * @return true if output has been canceled 618 */ 619 private synchronized boolean isCancelled() { 620 return cancelled; 621 } 622 623 /** 624 * Determines if the output is done. Access to the 625 * done variable should be within a synchronized block such 626 * as this to ensure that access is coherent. 627 * 628 * @return true if output has completed 629 */ 630 private synchronized boolean isDone() { 631 return done; 632 } 633 634 /** 635 * Provides a LineListener for this clas. 636 */ 637 private class JavaStreamLineListener implements LineListener { 638 639 /** 640 * Implements update() method of LineListener interface. Responds 641 * to the line events as appropriate. 642 * 643 * @param event the LineEvent to handle 644 */ 645 public void update(LineEvent event) { 646 if (event.getType().equals(LineEvent.Type.OPEN)) { 647 synchronized (openLock) { 648 openLock.notifyAll(); 649 } 650 } 651 } 652 } 653}