diff --git a/lect-core/src/main/java/org/lecturestudio/core/ExecutableBase.java b/lect-core/src/main/java/org/lecturestudio/core/ExecutableBase.java index 55b1b004..b01ac50f 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/ExecutableBase.java +++ b/lect-core/src/main/java/org/lecturestudio/core/ExecutableBase.java @@ -51,7 +51,7 @@ public abstract class ExecutableBase implements Executable { /** - * Add a ExecutableStateListener listener to this component. + * Adds an {@code ExecutableStateListener} to this component. * * @param listener The listener to add. */ @@ -60,7 +60,7 @@ public abstract class ExecutableBase implements Executable { } /** - * Remove a ExecutableStateListener listener from this component. + * Removes an {@code ExecutableStateListener} from this component. * * @param listener The listener to remove. */ diff --git a/lect-core/src/main/java/org/lecturestudio/core/app/configuration/AudioConfiguration.java b/lect-core/src/main/java/org/lecturestudio/core/app/configuration/AudioConfiguration.java index d9322348..aa0e9397 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/app/configuration/AudioConfiguration.java +++ b/lect-core/src/main/java/org/lecturestudio/core/app/configuration/AudioConfiguration.java @@ -19,6 +19,7 @@ package org.lecturestudio.core.app.configuration; import org.lecturestudio.core.audio.AudioFormat; +import org.lecturestudio.core.audio.AudioProcessingSettings; import org.lecturestudio.core.beans.DoubleProperty; import org.lecturestudio.core.beans.FloatProperty; import org.lecturestudio.core.beans.ObjectProperty; @@ -40,9 +41,6 @@ public class AudioConfiguration { /** The playback device name. */ private final StringProperty playbackDeviceName = new StringProperty(); - /** The sound system name. */ - private final StringProperty soundSystem = new StringProperty(); - /** The path where the recordings are stored at. */ private final StringProperty recordingPath = new StringProperty(); @@ -61,6 +59,9 @@ public class AudioConfiguration { /** The audio format of the recording. */ private final ObjectProperty recordingFormat = new ObjectProperty<>(); + /** The audio processing settings for recording. */ + private final ObjectProperty recordingProcessingSettings = new ObjectProperty<>(); + /** * Obtain the capture device name. @@ -116,33 +117,6 @@ public class AudioConfiguration { return playbackDeviceName; } - /** - * Obtain the sound system name. - * - * @return the sound system name. - */ - public String getSoundSystem() { - return soundSystem.get(); - } - - /** - * Set the sound system name. - * - * @param soundSystem sound system name to set. - */ - public void setSoundSystem(String soundSystem) { - this.soundSystem.set(soundSystem); - } - - /** - * Obtain the sound system property. - * - * @return the sound system property. - */ - public StringProperty soundSystemProperty() { - return soundSystem; - } - /** * Obtain the recording path. * @@ -314,4 +288,32 @@ public class AudioConfiguration { return recordingFormat; } + /** + * Obtain the {@code AudioProcessingSettings} for audio recording. + * + * @return the {@code AudioProcessingSettings} for recording. + */ + public AudioProcessingSettings getRecordingProcessingSettings() { + return recordingProcessingSettings.get(); + } + + /** + * Set the {@code AudioProcessingSettings} to be applied when recording + * audio. + * + * @param settings The new {@code AudioProcessingSettings}. + */ + public void setRecordingProcessingSettings(AudioProcessingSettings settings) { + this.recordingProcessingSettings.set(settings); + } + + /** + * Get the {@code AudioProcessingSettings} which are applied for audio + * recordings. + * + * @return The {@code AudioProcessingSettings}. + */ + public ObjectProperty recordingProcessingSettingsProperty() { + return recordingProcessingSettings; + } } diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlaybackProgressListener.java b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlaybackProgressListener.java index b61c0e69..77bc571d 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlaybackProgressListener.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlaybackProgressListener.java @@ -21,8 +21,8 @@ package org.lecturestudio.core.audio; import org.lecturestudio.core.model.Time; /** - * The {@link AudioPlaybackProgressListener} is notified each time an audio player has - * processed and pushed audio samples to a audio playback device. + * The {@link AudioPlaybackProgressListener} is notified each time an audio + * player has processed and pushed audio samples to a audio playback device. * * @author Alex Andres */ @@ -30,11 +30,11 @@ import org.lecturestudio.core.model.Time; public interface AudioPlaybackProgressListener { /** - * Called when progress of the current playing audio changes. + * Called when progress of the playing audio changes. * * @param progressMs The current progress in milliseconds. * @param durationMs The total duration of audio in milliseconds. */ void onAudioProgress(Time progressMs, Time durationMs); - + } diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlayer.java b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlayer.java index e4c7cc16..fb1b33c3 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlayer.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlayer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, * Embedded Systems and Applications Group. * * This program is free software: you can redistribute it and/or modify @@ -18,257 +18,70 @@ package org.lecturestudio.core.audio; -import static java.util.Objects.isNull; -import static java.util.Objects.nonNull; - -import org.lecturestudio.core.ExecutableBase; -import org.lecturestudio.core.ExecutableException; -import org.lecturestudio.core.ExecutableState; +import org.lecturestudio.core.Executable; import org.lecturestudio.core.ExecutableStateListener; -import org.lecturestudio.core.audio.device.AudioOutputDevice; import org.lecturestudio.core.audio.source.AudioSource; -import org.lecturestudio.core.model.Time; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; /** - * Default audio player implementation. + * An AudioPlayer manages the audio resources to play audio on an audio output + * device, e.g. a speaker or headset. * * @author Alex Andres */ -public class AudioPlayer extends ExecutableBase implements Player { - - /** Logger for {@link AudioPlayer} */ - private static final Logger LOG = LogManager.getLogger(AudioPlayer.class); - - /** The sync state that is shared with other media players. */ - private final SyncState syncState; - - /** The audio playback device. */ - private final AudioOutputDevice playbackDevice; - - /** The audio source. */ - private final AudioSource audioSource; - - /** The audio source size. */ - private final long inputSize; - - /** The playback progress listener. */ - private AudioPlaybackProgressListener progressListener; - - /** The player state listener. */ - private ExecutableStateListener stateListener; - - /** The playback thread. */ - private Thread thread; - - /** The current audio source reading position. */ - private long inputPos; - +public interface AudioPlayer extends Executable { /** - * Create an {@link AudioPlayer} with the specified playback device and source. The - * sync state is shared with other media players to keep different media - * sources in sync while playing. + * Sets the device name of the audio playback device which will play audio + * for this player. * - * @param device The audio playback device. - * @param source The audio source. - * @param syncState The shared sync state. - * - * @throws Exception If the audio player failed to initialize. + * @param deviceName The audio output device name. */ - public AudioPlayer(AudioOutputDevice device, AudioSource source, SyncState syncState) throws Exception { - if (isNull(device)) { - throw new NullPointerException("Missing audio playback device."); - } - if (isNull(source)) { - throw new NullPointerException("Missing audio source."); - } - - this.playbackDevice = device; - this.audioSource = source; - this.syncState = syncState; - this.inputSize = source.getInputSize(); - } + void setAudioDeviceName(String deviceName); - @Override - public void setVolume(float volume) { - if (volume < 0 || volume > 1) { - return; - } + /** + * Sets the {@code AudioSource} that will read the audio samples to play. + * + * @param source The audio source to set. + */ + void setAudioSource(AudioSource source); - playbackDevice.setVolume(volume); - } + /** + * Sets the recording audio volume. The value must be in the range of + * [0,1]. + * + * @param volume The recording audio volume. + */ + void setAudioVolume(double volume); - @Override - public void seek(int timeMs) throws Exception { - AudioFormat format = audioSource.getAudioFormat(); + /** + * Set the playback progress listener. + * + * @param listener The listener to set. + */ + void setAudioProgressListener(AudioPlaybackProgressListener listener); - float bytesPerSecond = AudioUtils.getBytesPerSecond(format); - int skipBytes = Math.round(bytesPerSecond * timeMs / 1000F); + /** + * Jump to the specified time position in the audio playback stream. + * + * @param timeMs The absolute time in milliseconds to jump to. + * + * @throws Exception If the playback stream failed to read the start of the + * specified position. + */ + void seek(int timeMs) throws Exception; - audioSource.reset(); - audioSource.skip(skipBytes); + /** + * Add an {@code ExecutableStateListener} to this player. + * + * @param listener The listener to add. + */ + void addStateListener(ExecutableStateListener listener); - inputPos = skipBytes; - - syncState.setAudioTime((long) (inputPos / (bytesPerSecond / 1000f))); - } - - @Override - public void setProgressListener(AudioPlaybackProgressListener listener) { - this.progressListener = listener; - } - - @Override - public void setStateListener(ExecutableStateListener listener) { - this.stateListener = listener; - } - - @Override - protected void initInternal() throws ExecutableException { - try { - audioSource.reset(); - } - catch (Exception e) { - throw new ExecutableException("Audio device could not be initialized.", e); - } - - if (!playbackDevice.supportsAudioFormat(audioSource.getAudioFormat())) { - throw new ExecutableException("Audio device does not support the needed audio format."); - } - - try { - playbackDevice.setAudioFormat(audioSource.getAudioFormat()); - playbackDevice.open(); - playbackDevice.start(); - } - catch (Exception e) { - throw new ExecutableException("Audio device could not be initialized.", e); - } - } - - @Override - protected void startInternal() throws ExecutableException { - if (getPreviousState() == ExecutableState.Suspended) { - synchronized (thread) { - thread.notify(); - } - } - else { - thread = new Thread(new AudioReaderTask(), getClass().getSimpleName()); - thread.start(); - } - } - - @Override - protected void stopInternal() throws ExecutableException { - try { - audioSource.reset(); - } - catch (Exception e) { - throw new ExecutableException(e); - } - - inputPos = 0; - syncState.reset(); - } - - @Override - protected void destroyInternal() throws ExecutableException { - try { - playbackDevice.close(); - audioSource.close(); - } - catch (Exception e) { - throw new ExecutableException(e); - } - } - - @Override - protected void fireStateChanged() { - if (nonNull(stateListener)) { - stateListener.onExecutableStateChange(getPreviousState(), getState()); - } - } - - private void onProgress(Time progress, Time duration, long progressMs) { - if (nonNull(syncState)) { - syncState.setAudioTime(progressMs); - } - if (nonNull(progressListener) && started()) { - progress.setMillis(progressMs); - - progressListener.onAudioProgress(progress, duration); - } - } - - - - private class AudioReaderTask implements Runnable { - - @Override - public void run() { - byte[] buffer = new byte[playbackDevice.getBufferSize()]; - int bytesRead; - - // Calculate bytes per millisecond. - float bpms = AudioUtils.getBytesPerSecond(audioSource.getAudioFormat()) / 1000f; - - Time progress = new Time(0); - Time duration = new Time((long) (inputSize / bpms)); - - ExecutableState state; - - while (true) { - state = getState(); - - if (state == ExecutableState.Started) { - try { - bytesRead = audioSource.read(buffer, 0, buffer.length); - - if (bytesRead > 0) { - playbackDevice.write(buffer, 0, bytesRead); - - inputPos += bytesRead; - - onProgress(progress, duration, (long) (inputPos / bpms)); - } - else if (bytesRead == -1) { - // EOM - break; - } - } - catch (Exception e) { - LOG.error("Play audio failed.", e); - break; - } - } - else if (state == ExecutableState.Suspended) { - synchronized (thread) { - try { - thread.wait(); - } - catch (Exception e) { - // Ignore - } - } - } - else if (state == ExecutableState.Stopped) { - return; - } - } - - // EOM - try { - stop(); - } - catch (ExecutableException e) { - LOG.error("Stop " + getClass().getName() + " failed.", e); - } - } - - } + /** + * Removes an {@code ExecutableStateListener} from this player. + * + * @param listener The listener to remove. + */ + void removeStateListener(ExecutableStateListener listener); } diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlayerExt.java b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlayerExt.java deleted file mode 100644 index ad655a89..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioPlayerExt.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio; - -import static java.util.Objects.isNull; -import static java.util.Objects.nonNull; - -import java.security.InvalidParameterException; - -import org.lecturestudio.core.ExecutableBase; -import org.lecturestudio.core.ExecutableException; -import org.lecturestudio.core.ExecutableState; -import org.lecturestudio.core.ExecutableStateListener; -import org.lecturestudio.core.audio.device.AudioOutputDevice; -import org.lecturestudio.core.audio.io.AudioPlaybackBuffer; -import org.lecturestudio.core.io.PlaybackData; -import org.lecturestudio.core.net.Synchronizer; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * Extended audio player implementation. - * - * @author Alex Andres - */ -public class AudioPlayerExt extends ExecutableBase implements Player { - - /** Logger for {@link AudioPlayerExt} */ - private static final Logger LOG = LogManager.getLogger(AudioPlayerExt.class); - - /** The audio playback device. */ - private AudioOutputDevice playbackDevice; - - /** The audio source. */ - private AudioPlaybackBuffer audioSource; - - /** The player state listener. */ - private ExecutableStateListener stateListener; - - /** The playback thread. */ - private Thread thread; - - - /** - * Create an {@link AudioPlayerExt} with the specified playback device and audio buffer. - * - * @param device The audio playback device. - * @param buffer The audio source. - */ - public AudioPlayerExt(AudioOutputDevice device, AudioPlaybackBuffer buffer) { - if (isNull(device)) { - throw new NullPointerException("Missing audio playback device."); - } - if (isNull(buffer)) { - throw new NullPointerException("Missing audio buffer source."); - } - - this.playbackDevice = device; - this.audioSource = buffer; - } - - @Override - public void setVolume(float volume) { - if (volume < 0 || volume > 1) { - throw new InvalidParameterException("Volume value should be within 0 and 1."); - } - - playbackDevice.setVolume(volume); - } - - @Override - public void seek(int time) { - audioSource.skip(time); - } - - @Override - public void setProgressListener(AudioPlaybackProgressListener listener) { - - } - - @Override - public void setStateListener(ExecutableStateListener listener) { - this.stateListener = listener; - } - - @Override - protected void initInternal() throws ExecutableException { - audioSource.reset(); - - AudioFormat format = audioSource.getAudioFormat(); - - if (!playbackDevice.supportsAudioFormat(format)) { - throw new ExecutableException("Audio device does not support the needed audio format."); - } - - try { - playbackDevice.setAudioFormat(format); - playbackDevice.open(); - playbackDevice.start(); - } - catch (Exception e) { - throw new ExecutableException(e); - } - } - - @Override - protected void startInternal() throws ExecutableException { - if (getPreviousState() == ExecutableState.Suspended) { - synchronized (thread) { - thread.notify(); - } - } - else { - thread = new Thread(new AudioReaderTask(), getClass().getSimpleName()); - thread.start(); - } - } - - @Override - protected void stopInternal() throws ExecutableException { - try { - synchronized (thread) { - thread.interrupt(); - } - - audioSource.reset(); - } - catch (Exception e) { - throw new ExecutableException("Audio source could not be reset.", e); - } - } - - @Override - protected void destroyInternal() throws ExecutableException { - try { - playbackDevice.close(); - audioSource.reset(); - } - catch (Exception e) { - throw new ExecutableException(e); - } - } - - @Override - protected void fireStateChanged() { - if (nonNull(stateListener)) { - stateListener.onExecutableStateChange(getPreviousState(), getState()); - } - } - - - - private class AudioReaderTask implements Runnable { - - @Override - public void run() { - int bytesRead; - ExecutableState state; - - while (playbackDevice.isOpen()) { - state = getState(); - - if (state == ExecutableState.Started) { - try { - PlaybackData samples = audioSource.take(); - - if (nonNull(samples)) { - bytesRead = samples.getData().length; - - Synchronizer.setAudioTime(samples.getTimestamp()); - - if (bytesRead > 0 && playbackDevice.isOpen()) { - playbackDevice.write(samples.getData(), 0, bytesRead); // TODO check deviceBufferSize - } - } - } - catch (Exception e) { - LOG.error("Play audio failed.", e); - break; - } - } - else if (state == ExecutableState.Suspended) { - synchronized (thread) { - try { - thread.wait(); - } - catch (Exception e) { - // Ignore - } - } - } - else if (state == ExecutableState.Stopped) { - return; - } - } - - try { - playbackDevice.stop(); - playbackDevice.close(); - } - catch (Exception e) { - LOG.error("Stop audio playback device failed.", e); - } - } - - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioProcessingSettings.java b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioProcessingSettings.java new file mode 100644 index 00000000..3416de31 --- /dev/null +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioProcessingSettings.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.core.audio; + +import org.lecturestudio.core.beans.BooleanProperty; +import org.lecturestudio.core.beans.ObjectProperty; + +/** + * Specifies software audio processing filters to be applied to audio samples + * coming from an audio input device, e.g. a microphone, or being provided to an + * audio output device, e.g. speakers. If the hardware has activated such + * filters, then the corresponding setting should be disabled here. + * + * @author Alex Andres + */ +public class AudioProcessingSettings { + + public enum NoiseSuppressionLevel { + LOW, + MODERATE, + HIGH, + VERY_HIGH + } + + + + private final BooleanProperty enableEchoCanceller = new BooleanProperty(); + + private final BooleanProperty enableGainControl = new BooleanProperty(); + + private final BooleanProperty enableHighpassFilter = new BooleanProperty(); + + private final BooleanProperty enableNoiseSuppression = new BooleanProperty(); + + private final BooleanProperty enableLevelEstimation = new BooleanProperty(); + + private final BooleanProperty enableVoiceDetection = new BooleanProperty(); + + private final ObjectProperty noiseSuppressionLevel = new ObjectProperty<>(); + + + public boolean isEchoCancellerEnabled() { + return enableEchoCanceller.get(); + } + + public void setEchoCancellerEnabled(boolean enable) { + enableEchoCanceller.set(enable); + } + + public boolean isGainControlEnabled() { + return enableGainControl.get(); + } + + public void setGainControlEnabled(boolean enable) { + enableGainControl.set(enable); + } + + public boolean isHighpassFilterEnabled() { + return enableHighpassFilter.get(); + } + + public void setHighpassFilterEnabled(boolean enable) { + enableHighpassFilter.set(enable); + } + + public boolean isNoiseSuppressionEnabled() { + return enableNoiseSuppression.get(); + } + + public void setNoiseSuppressionEnabled(boolean enable) { + enableNoiseSuppression.set(enable); + } + + public boolean isLevelEstimationEnabled() { + return enableLevelEstimation.get(); + } + + public void setLevelEstimationEnabled(boolean enable) { + enableLevelEstimation.set(enable); + } + + public boolean isVoiceDetectionEnabled() { + return enableVoiceDetection.get(); + } + + public void setVoiceDetectionEnabled(boolean enable) { + enableVoiceDetection.set(enable); + } + + public NoiseSuppressionLevel getNoiseSuppressionLevel() { + return noiseSuppressionLevel.get(); + } + + public void setNoiseSuppressionLevel(NoiseSuppressionLevel level) { + noiseSuppressionLevel.set(level); + } + + public ObjectProperty noiseSuppressionLevelProperty() { + return noiseSuppressionLevel; + } +} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioProcessingStats.java b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioProcessingStats.java new file mode 100644 index 00000000..46e6ec44 --- /dev/null +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioProcessingStats.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.core.audio; + +/** + * Audio statistics produced by an audio processing module. + * + * @author Alex Andres + */ +public class AudioProcessingStats { + + /** + * The root-mean-square (RMS) level in dBFS (decibels from digital + * full-scale) of the last capture frame, after processing. It is + * constrained to [-127, 0]. + *

+ * The computation follows: https://tools.ietf.org/html/rfc6465 with the + * intent that it can provide the RTP audio level indication. + *

+ * Only reported if level estimation is enabled via {@code + * AudioProcessingSettings}. + */ + public int outputRmsDbfs; + + /** + * True if voice is detected in the last capture frame, after processing. + *

+ * It is conservative in flagging audio as speech, with low likelihood of + * incorrectly flagging a frame as voice. Only reported if voice detection + * is enabled via {@code AudioProcessingSettings}. + */ + public boolean voiceDetected; + + /** + * The instantaneous delay estimate produced in the AEC. The unit is in + * milliseconds and the value is the instantaneous value at the time of the + * call to getStatistics(). + */ + public int delayMs; + +} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioRecorder.java b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioRecorder.java new file mode 100644 index 00000000..340bc722 --- /dev/null +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioRecorder.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.core.audio; + +import org.lecturestudio.core.Executable; +import org.lecturestudio.core.ExecutableStateListener; +import org.lecturestudio.core.audio.sink.AudioSink; + +/** + * An AudioRecorder manages the audio resources to record audio from an audio + * input device, e.g. a microphone. + * + * @author Alex Andres + */ +public interface AudioRecorder extends Executable { + + /** + * Sets the device name of the audio recording device which will capture + * audio for this recorder. + * + * @param deviceName The audio capture device name. + */ + void setAudioDeviceName(String deviceName); + + /** + * Sets the {@code AudioSink} that will receive the captured audio samples. + * + * @param sink The audio sink to set. + */ + void setAudioSink(AudioSink sink); + + /** + * Sets the recording audio volume. The value must be in the range of + * [0,1]. + * + * @param volume The recording audio volume. + */ + void setAudioVolume(double volume); + + /** + * Get audio processing statistics. This method will only return valid + * statistics if {@link #setAudioProcessingSettings} has been called prior + * recording. + * + * @return The audio processing statistics. + */ + AudioProcessingStats getAudioProcessingStats(); + + /** + * Sets which software audio processing filters to be applied to recorded + * audio samples. + * + * @param settings The {@code AudioProcessingSettings} to be applied. + */ + void setAudioProcessingSettings(AudioProcessingSettings settings); + + /** + * Add an {@code ExecutableStateListener} to this player. + * + * @param listener The listener to add. + */ + void addStateListener(ExecutableStateListener listener); + + /** + * Removes an {@code ExecutableStateListener} from this player. + * + * @param listener The listener to remove. + */ + void removeStateListener(ExecutableStateListener listener); + +} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioSystemProvider.java b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioSystemProvider.java new file mode 100644 index 00000000..225ffa4e --- /dev/null +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioSystemProvider.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.core.audio; + +import org.lecturestudio.core.audio.device.AudioDevice; + +/** + * Audio system provider implementation. This {@code AudioSystemProvider} + * provides access to all connected audio devices, such as microphones and + * speakers. + * + * @author Alex Andres + */ +public interface AudioSystemProvider { + + /** + * Get the systems default audio recording device. + * + * @return The default audio recording device. + */ + AudioDevice getDefaultRecordingDevice(); + + /** + * Get the systems default audio playback device. + * + * @return The default audio playback device. + */ + AudioDevice getDefaultPlaybackDevice(); + + /** + * Get all available audio recording devices. + * + * @return An array of all audio recording devices. + */ + AudioDevice[] getRecordingDevices(); + + /** + * Get all available audio playback devices. + * + * @return An array of all audio playback devices. + */ + AudioDevice[] getPlaybackDevices(); + + /** + * Creates an audio player based on this provider internal implementation. + * + * @return A new audio player. + */ + AudioPlayer createAudioPlayer(); + + /** + * Creates an audio recorder based on this provider internal + * implementation. + * + * @return A new audio recorder. + */ + AudioRecorder createAudioRecorder(); + + /** + * Get the implementing service provider's name. + * + * @return the name of the service provider. + */ + String getProviderName(); + +} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioUtils.java b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioUtils.java index 685b8b16..6d4bea39 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/AudioUtils.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/AudioUtils.java @@ -18,15 +18,10 @@ package org.lecturestudio.core.audio; -import static java.util.Objects.isNull; +import java.util.ArrayList; +import java.util.List; -import java.util.Arrays; - -import org.lecturestudio.core.audio.codec.AudioCodecLoader; -import org.lecturestudio.core.audio.device.AudioInputDevice; -import org.lecturestudio.core.audio.device.AudioOutputDevice; -import org.lecturestudio.core.audio.system.AudioSystemLoader; -import org.lecturestudio.core.audio.system.AudioSystemProvider; +import org.lecturestudio.core.audio.AudioFormat.Encoding; /** * Audio-related utility methods. @@ -35,165 +30,31 @@ import org.lecturestudio.core.audio.system.AudioSystemProvider; */ public class AudioUtils { - /** the singleton instance of {@link AudioSystemLoader} */ - private static final AudioSystemLoader LOADER = AudioSystemLoader.getInstance(); + /** An array of all available sample rates to support. */ + public static final int[] SUPPORTED_SAMPLE_RATES = new int[] { + 8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000, 96000 + }; + /** - * Get default audio capture device of the {@link AudioSystemProvider} with the specified name. + * Returns all supported {@code AudioFormat}s which can be used for audio + * playback or recording. * - * @param providerName The name of the {@link AudioSystemProvider}. - * @return The default audio capture device of the {@link AudioSystemProvider} - * or null if the{@link AudioSystemProvider} could not be found. + * @return A list of all supported {@code AudioFormat}s. */ - public static AudioInputDevice getDefaultAudioCaptureDevice(String providerName) { - AudioSystemProvider provider = LOADER.getProvider(providerName); + public static List getAudioFormats() { + List formats = new ArrayList<>(); - return isNull(provider) ? null : provider.getDefaultInputDevice(); - } - - /** - * Get default audio playback device of the {@link AudioSystemProvider} with the specified name. - * - * @param providerName The name of the {@link AudioSystemProvider}. - * @return The default audio playback device of the {@link AudioSystemProvider} - * or null if the{@link AudioSystemProvider} could not be found.. - */ - public static AudioOutputDevice getDefaultAudioPlaybackDevice(String providerName) { - AudioSystemProvider provider = LOADER.getProvider(providerName); - - return isNull(provider) ? null : provider.getDefaultOutputDevice(); - } - - /** - * Get all available audio capture devices of the {@link AudioSystemProvider} with the specified name. - * - * @param providerName The name of the {@link AudioSystemProvider}. - * @return All available audio capture devices of the {@link AudioSystemProvider}. - */ - public static AudioInputDevice[] getAudioCaptureDevices(String providerName) { - AudioSystemProvider provider = LOADER.getProvider(providerName); - - return isNull(provider) ? new AudioInputDevice[0] : provider.getInputDevices(); - } - - /** - * Get all available audio playback devices of the {@link AudioSystemProvider} with the specified name. - * - * @param providerName The name of the {@link AudioSystemProvider}. - * @return All available audio playback devices of the {@link AudioSystemProvider}. - */ - public static AudioOutputDevice[] getAudioPlaybackDevices(String providerName) { - AudioSystemProvider provider = LOADER.getProvider(providerName); - - return isNull(provider) ? new AudioOutputDevice[0] : provider.getOutputDevices(); - } - - /** - * Checks if an available audio capture device of the {@link - * AudioSystemProvider} with the {@code providerName} has the same name as - * the specified {@code deviceName}. - * - * @param providerName The name of the {@link AudioSystemProvider}. - * @param deviceName The name of the device. - * - * @return {@code true} if an available audio capture device has the same - * name as the specified {@code deviceName}, otherwise {@code false}. - */ - public static boolean hasCaptureDevice(String providerName, String deviceName) { - if (isNull(deviceName)) { - return false; + for (int sampleRate : SUPPORTED_SAMPLE_RATES) { + formats.add(new AudioFormat(Encoding.S16LE, sampleRate, 1)); } - return Arrays.stream(getAudioCaptureDevices(providerName)) - .anyMatch(device -> device.getName().equals(deviceName)); + return formats; } /** - * Checks if an available audio playback device of the {@link - * AudioSystemProvider} with the {@code providerName} has the same name as - * the specified {@code deviceName}. - * - * @param providerName The name of the {@link AudioSystemProvider}. - * @param deviceName The name of the device. - * - * @return {@code true} if an available audio playback device has the same - * name as the specified {@code deviceName}, otherwise {@code false}. - */ - public static boolean hasPlaybackDevice(String providerName, String deviceName) { - if (isNull(deviceName)) { - return false; - } - - return Arrays.stream(getAudioPlaybackDevices(providerName)) - .anyMatch(device -> device.getName().equals(deviceName)); - } - - /** - * Get an {@link AudioInputDevice} with the specified device name that is registered - * with the given audio system provider. - * - * @param providerName The audio system provider name. - * @param deviceName The audio capture device name. - * - * @return the retrieved {@link AudioInputDevice} or null if the capture device could not be found. - */ - public static AudioInputDevice getAudioInputDevice(String providerName, String deviceName) { - AudioSystemProvider provider = LOADER.getProvider(providerName); - - if (isNull(provider)) { - throw new NullPointerException("Audio provider is not available: " + providerName); - } - - AudioInputDevice inputDevice = provider.getInputDevice(deviceName); - - if (isNull(inputDevice)) { - throw new NullPointerException("Audio device is not available: " + deviceName); - } - - return inputDevice; - } - - /** - * Get an {@link AudioOutputDevice} with the specified device name that is - * registered with the given audio system provider. - * - * @param providerName The audio system provider name. - * @param deviceName The audio playback device name. - * - * @return the retrieved {@link AudioOutputDevice} or null if the playback device could not be found. - */ - public static AudioOutputDevice getAudioOutputDevice(String providerName, String deviceName) { - AudioSystemProvider provider = LOADER.getProvider(providerName); - - if (isNull(provider)) { - provider = LOADER.getProvider("Java Sound"); - } - - AudioOutputDevice outputDevice = provider.getOutputDevice(deviceName); - - if (outputDevice == null) { - // Get next best device. - for (AudioOutputDevice device : provider.getOutputDevices()) { - if (device != null) { - return device; - } - } - } - - return outputDevice; - } - - /** - * Retrieve all supported audio codecs by the system. - * - * @return an array of names of supported audio codecs. - */ - public static String[] getSupportedAudioCodecs() { - return AudioCodecLoader.getInstance().getProviderNames(); - } - - /** - * Compute the number of bytes per second that the specified audio format will require. + * Computes the number of bytes per second that the specified audio format + * will require. * * @param audioFormat The audio format. * @@ -205,7 +66,8 @@ public class AudioUtils { } /** - * Pack two sequential bytes into a {@code short} value according to the specified endianness. + * Pack two sequential bytes into a {@code short} value according to the + * specified endianness. * * @param bytes The bytes to pack, must of size 2. * @param bigEndian True to pack with big-endian order, false to pack with @@ -238,15 +100,17 @@ public class AudioUtils { } /** - * Convert the specified integer value to a normalized float value in the range of [0,1]. + * Convert the specified integer value to a normalized float value in the + * range of [0,1]. * - * @param value The integer value to convert. + * @param value The integer value to convert. * @param frameSize The sample size in bytes. - * @param signed True to respect the sign bit, false otherwise. + * @param signed True to respect the sign bit, false otherwise. * * @return a normalized float value. */ - public static float getNormalizedSampleValue(int value, int frameSize, boolean signed) { + public static float getNormalizedSampleValue(int value, int frameSize, + boolean signed) { float relValue; int maxValue; diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/Player.java b/lect-core/src/main/java/org/lecturestudio/core/audio/Player.java deleted file mode 100644 index 8efddf56..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/Player.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio; - -import org.lecturestudio.core.Executable; -import org.lecturestudio.core.ExecutableStateListener; - -/** - * Common interface to provide a consistent mechanism for media players. - * - * @author Alex Andres - */ -public interface Player extends Executable { - - /** - * Set the audio volume for playback. The volume value must be in the range of [0,1]. - * - * @param volume The new volume value. - */ - void setVolume(float volume); - - /** - * Jump to the specified time position in the audio playback stream. - * - * @param timeMs The absolute time in milliseconds to jump to. - * - * @throws Exception If the playback stream failed to read the start of the - * specified position. - */ - void seek(int timeMs) throws Exception; - - /** - * Set the playback progress listener. - * - * @param listener The listener to set. - */ - void setProgressListener(AudioPlaybackProgressListener listener); - - /** - * Set the state listener. - * - * @param listener The listener to set. - */ - void setStateListener(ExecutableStateListener listener); - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/RingBuffer.java b/lect-core/src/main/java/org/lecturestudio/core/audio/RingBuffer.java index db208b99..cd4b2b85 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/RingBuffer.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/RingBuffer.java @@ -205,6 +205,11 @@ public class RingBuffer implements AudioSink, AudioSource { return 0; } + @Override + public int seekMs(int timeMs) { + return 0; + } + @Override public long getInputSize() { return 0; diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/codec/OpusCodecProvider.java b/lect-core/src/main/java/org/lecturestudio/core/audio/codec/OpusCodecProvider.java deleted file mode 100644 index ce8b4c57..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/codec/OpusCodecProvider.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.codec; - -import com.github.javaffmpeg.CodecID; - -import org.lecturestudio.core.audio.codec.ffmpeg.FFmpegAudioDecoder; -import org.lecturestudio.core.audio.codec.ffmpeg.FFmpegAudioEncoder; -import org.lecturestudio.core.audio.codec.ffmpeg.FFmpegRtpDepacketizer; -import org.lecturestudio.core.audio.codec.ffmpeg.FFmpegRtpPacketizer; -import org.lecturestudio.core.net.rtp.RtpDepacketizer; -import org.lecturestudio.core.net.rtp.RtpPacketizer; - -/** - * OPUS audio codec provider implementation. - * - * @link http://opus-codec.org - * - * @author Alex Andres - */ -public class OpusCodecProvider implements AudioCodecProvider { - - @Override - public AudioEncoder getAudioEncoder() { - return new FFmpegAudioEncoder(CodecID.OPUS); - } - - @Override - public AudioDecoder getAudioDecoder() { - return new FFmpegAudioDecoder(CodecID.OPUS); - } - - @Override - public RtpPacketizer getRtpPacketizer() { - return new FFmpegRtpPacketizer(); - } - - @Override - public RtpDepacketizer getRtpDepacketizer() { - return new FFmpegRtpDepacketizer(); - } - - @Override - public String getProviderName() { - return "OPUS"; - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegAudioDecoder.java b/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegAudioDecoder.java deleted file mode 100644 index 386313e5..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegAudioDecoder.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.codec.ffmpeg; - -import com.github.javaffmpeg.Audio; -import com.github.javaffmpeg.AudioFrame; -import com.github.javaffmpeg.AudioResampler; -import com.github.javaffmpeg.Codec; -import com.github.javaffmpeg.CodecID; -import com.github.javaffmpeg.Decoder; -import com.github.javaffmpeg.JavaFFmpegException; -import com.github.javaffmpeg.MediaPacket; -import com.github.javaffmpeg.MediaType; -import com.github.javaffmpeg.SampleFormat; - -import java.nio.ByteBuffer; - -import org.lecturestudio.core.ExecutableException; -import org.lecturestudio.core.audio.AudioFormat; -import org.lecturestudio.core.audio.codec.AudioDecoder; - -import org.bytedeco.javacpp.BytePointer; - -/** - * FFmpeg audio decoder implementation. - * - * @link https://ffmpeg.org - * - * @author Alex Andres - */ -public class FFmpegAudioDecoder extends AudioDecoder { - - /** The sample size in bytes. */ - private static final int SAMPLE_SIZE = 2; - - /** The internal FFmpeg decoder. */ - private Decoder decoder; - - /** The internal audio resampler */ - private com.github.javaffmpeg.AudioResampler resampler; - - - /** - * Create a FFmpegAudioDecoder with the specified codec ID. Based on the ID - * the corresponding FFmpeg decoder will be created. - * - * @param codecID The ID of the codec to use. - */ - public FFmpegAudioDecoder(CodecID codecID) { - try { - decoder = new Decoder(Codec.getDecoderById(codecID)); - decoder.setMediaType(MediaType.AUDIO); - } - catch (JavaFFmpegException e) { - e.printStackTrace(); - } - } - - @Override - public void process(byte[] input, int length, long timestamp) throws Exception { - ByteBuffer buffer = ByteBuffer.wrap(input, 0, length); - MediaPacket packet = new MediaPacket(buffer); - - AudioFrame frame = decoder.decodeAudio(packet); - - if (frame != null) { - if (resampler != null) { - AudioFrame[] frames = resampler.resample(frame); - - for (AudioFrame resFrame : frames) { - processAudioFrame(resFrame, timestamp); - - resFrame.clear(); - } - } - else { - processAudioFrame(frame, timestamp); - } - - frame.clear(); - } - - packet.clear(); - } - - @Override - protected void initInternal() throws ExecutableException { - - } - - @Override - protected void startInternal() throws ExecutableException { - AudioFormat inputFormat = getFormat(); - - int sampleRate = inputFormat.getSampleRate(); - int channels = inputFormat.getChannels(); - - try { - decoder.setSampleRate(sampleRate); - decoder.setSampleFormat(SampleFormat.S16); - decoder.setAudioChannels(channels); - decoder.open(null); - } - catch (Exception e) { - throw new ExecutableException(e); - } - - // requested format - com.github.javaffmpeg.AudioFormat reqFormat = new com.github.javaffmpeg.AudioFormat(); - reqFormat.setChannelLayout(Audio.getChannelLayout(channels)); - reqFormat.setChannels(channels); - reqFormat.setSampleFormat(SampleFormat.S16); - reqFormat.setSampleRate(sampleRate); - - // decoder format - com.github.javaffmpeg.AudioFormat decFormat = new com.github.javaffmpeg.AudioFormat(); - decFormat.setChannelLayout(decoder.getChannelLayout()); - decFormat.setChannels(decoder.getAudioChannels()); - decFormat.setSampleFormat(decoder.getSampleFormat()); - decFormat.setSampleRate(decoder.getSampleRate()); - - // in some cases the decoder chooses its own parameters, e.g. OPUS - if (!reqFormat.equals(decFormat)) { - int samples = sampleRate / 50; // 20 ms audio - resampler = new AudioResampler(); - - try { - resampler.open(decFormat, reqFormat, samples); - } - catch (Exception e) { - throw new ExecutableException(e); - } - } - } - - @Override - protected void stopInternal() throws ExecutableException { - decoder.close(); - resampler.close(); - } - - @Override - protected void destroyInternal() throws ExecutableException { - - } - - private void processAudioFrame(AudioFrame frame, long timestamp) { - int planes = frame.getPlaneCount(); - int size = planes * frame.getPlane(0).limit(); - byte[] samples = new byte[size]; - - // interleave planes - for (int i = 0; i < planes; i++) { - BytePointer plane = frame.getPlane(i); - ByteBuffer pBuffer = plane.asByteBuffer(); - int pLength = plane.limit(); - int offset = i * planes; - - for (int j = 0, k = offset; j < pLength; j += SAMPLE_SIZE) { - samples[k++] = (byte) (pBuffer.get() & 0xFF); - samples[k++] = (byte) (pBuffer.get() & 0xFF); - - k += SAMPLE_SIZE * (planes - 1); - } - } - - fireAudioDecoded(samples, samples.length, timestamp); - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegAudioEncoder.java b/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegAudioEncoder.java deleted file mode 100644 index e00e6ec5..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegAudioEncoder.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.codec.ffmpeg; - -import com.github.javaffmpeg.Audio; -import com.github.javaffmpeg.AudioFrame; -import com.github.javaffmpeg.Codec; -import com.github.javaffmpeg.CodecID; -import com.github.javaffmpeg.Encoder; -import com.github.javaffmpeg.JavaFFmpegException; -import com.github.javaffmpeg.MediaPacket; -import com.github.javaffmpeg.MediaType; -import com.github.javaffmpeg.SampleFormat; - -import org.lecturestudio.core.ExecutableException; -import org.lecturestudio.core.audio.AudioFormat; -import org.lecturestudio.core.audio.codec.AudioEncoder; - -/** - * FFmpeg audio encoder implementation. - * - * @link https://ffmpeg.org - * - * @author Alex Andres - */ -public class FFmpegAudioEncoder extends AudioEncoder { - - /** An array of supported audio formats. */ - private AudioFormat[] supportedFormats; - - /** The internal FFmpeg encoder. */ - private Encoder encoder; - - /** The internal encoding format. */ - private com.github.javaffmpeg.AudioFormat format; - - /** The sample size in bytes. */ - private int sampleSize; - - - /** - * Create a {@link FFmpegAudioEncoder} with the specified codec ID. Based on the ID - * the corresponding FFmpeg encoder will be created. - * - * @param codecId The ID of the codec to use. - */ - public FFmpegAudioEncoder(CodecID codecId) { - try { - encoder = new Encoder(Codec.getEncoderById(codecId)); - encoder.setMediaType(MediaType.AUDIO); - - setBitrate(128000); - - getInputFormats(); - } - catch (JavaFFmpegException e) { - e.printStackTrace(); - } - } - - @Override - public AudioFormat[] getSupportedFormats() { - return supportedFormats; - } - - @Override - public void process(byte[] input, int length, long timestamp) throws Exception { - int samples = input.length / sampleSize; - - AudioFrame frame = new AudioFrame(format, samples); - - // mono stream - if (format.getChannels() == 1) { - frame.getPlane(0).asByteBuffer().put(input); - } - else { - throw new Exception("Frame input only for mono audio implemented."); - } - - MediaPacket[] packets = encoder.encodeAudio(frame); - - if (packets != null) { - for (MediaPacket packet : packets) { - if (packet == null) { - continue; - } - - byte[] outputData = new byte[packet.getData().limit()]; - packet.getData().get(outputData); - - fireAudioEncoded(outputData, outputData.length, timestamp); - - packet.clear(); - } - } - - frame.clear(); - } - - @Override - protected void initInternal() throws ExecutableException { - - } - - @Override - protected void startInternal() throws ExecutableException { - AudioFormat inputFormat = getFormat(); - - int sampleRate = inputFormat.getSampleRate(); - int channels = inputFormat.getChannels(); - - try { - encoder.setMediaType(MediaType.AUDIO); - encoder.setBitrate(getBitrate()); - encoder.setSampleRate(sampleRate); - encoder.setSampleFormat(SampleFormat.S16); - encoder.setAudioChannels(channels); - encoder.setQuality(0); // Quality-based encoding not supported. - encoder.open(null); - } - catch (JavaFFmpegException e) { - throw new ExecutableException(e); - } - - format = new com.github.javaffmpeg.AudioFormat(); - format.setChannelLayout(Audio.getChannelLayout(channels)); - format.setChannels(channels); - format.setSampleFormat(SampleFormat.S16); - format.setSampleRate(sampleRate); - - sampleSize = 2 * format.getChannels(); - } - - @Override - protected void stopInternal() throws ExecutableException { - encoder.close(); - } - - @Override - protected void destroyInternal() throws ExecutableException { - - } - - /** - * Assigns all supported audio formats to {@link #supportedFormats}. - */ - private void getInputFormats() { - if (encoder == null) { - return; - } - - Integer[] sampleRates = encoder.getCodec().getSupportedSampleRates(); - - if (sampleRates == null) { - return; - } - - supportedFormats = new AudioFormat[sampleRates.length]; - - AudioFormat.Encoding encoding = AudioFormat.Encoding.S16LE; - int channels = 1; - - for (int i = 0; i < sampleRates.length; i++) { - int sampleRate = sampleRates[i]; - - AudioFormat audioFormat = new AudioFormat(encoding, sampleRate, channels); - - supportedFormats[i] = audioFormat; - } - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegRtpPacketizer.java b/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegRtpPacketizer.java deleted file mode 100644 index 427ad108..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/codec/ffmpeg/FFmpegRtpPacketizer.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.codec.ffmpeg; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import org.lecturestudio.core.net.rtp.RtpPacket; -import org.lecturestudio.core.net.rtp.RtpPacketizer; - -/** - * FFmpeg RTP packetizer implementation. - * - * @author Alex Andres - */ -public class FFmpegRtpPacketizer implements RtpPacketizer { - - /** The RTP packet which should be sent. */ - private RtpPacket rtpPacket; - - - /** - * Create a new FFmpegRtpPacketizer instance and set the default RTP packet - * header values. - */ - public FFmpegRtpPacketizer() { - Random rand = new Random(); - - rtpPacket = new RtpPacket(); - rtpPacket.setVersion(2); - rtpPacket.setPadding(0); - rtpPacket.setExtension(0); - - rtpPacket.setMarker(0); - rtpPacket.setPayloadType(97); // dynamic - - rtpPacket.setSeqNumber(rand.nextInt()); - rtpPacket.setTimestamp(rand.nextInt()); - rtpPacket.setSsrc(rand.nextInt()); - } - - @Override - public List processPacket(byte[] payload, int payloadLength, long timestamp) { - List packets = new ArrayList<>(); - - updateRtpPacket(timestamp); - packPacket(payload, payloadLength); - - packets.add(rtpPacket.clone()); - - return packets; - } - - /** - * Sets the payload of {@link #rtpPacket} - * - * @param packetBytes The payload data to pack - * @param payloadSize The length of the payload data - */ - private void packPacket(byte[] packetBytes, int payloadSize) { - byte[] payload = new byte[payloadSize]; - - System.arraycopy(packetBytes, 0, payload, 0, payloadSize); - - rtpPacket.setPayload(payload); - } - - /** - * Sets the sequence number and timestamp of {@link #rtpPacket}. - * - * @param timestamp The new timestamp of {@link #rtpPacket} - */ - private void updateRtpPacket(long timestamp) { - /* Increment RTP header flags */ - rtpPacket.setSeqNumber(rtpPacket.getSeqNumber() + 1); - rtpPacket.setTimestamp(timestamp); - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioDevice.java b/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioDevice.java index de1033a4..9b6b8518 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioDevice.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioDevice.java @@ -18,260 +18,33 @@ package org.lecturestudio.core.audio.device; -import java.util.Arrays; -import java.util.List; - -import org.lecturestudio.core.audio.AudioFormat; - /** - * Common class to provide a consistent mechanism for audio devices. + * Common class to provide a consistent interface to audio devices. * * @author Alex Andres */ -public abstract class AudioDevice { +public class AudioDevice { - /** An array of all available sample rates to support. */ - public static final int[] SUPPORTED_SAMPLE_RATES = new int[] { - 8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000 - }; + /** The name of this audio device. */ + private final String name; - /** The audio format to be used by the audio device. */ - private AudioFormat audioFormat; - - /** The audio volume for playback or recording. */ - private double volume = 1; - - /** The measured audio signal power level of the last processed audio chunk. */ - private double signalPowerLevel = 0; - - /** Indicated whether the audio is muted or not. */ - private boolean mute = false; + /** + * Creates a new {@code AudioDevice} with the specified name. + * + * @param name The name of the device. + */ + public AudioDevice(String name) { + this.name = name; + } /** * Get the name of the audio device assigned by the operating system. * * @return the name of the audio device. */ - abstract public String getName(); - - /** - * Open the audio device and prepare the device to capture or play audio. - * - * @throws Exception If the audio device failed to open. - */ - abstract public void open() throws Exception; - - /** - * Close the audio device and release all previously assigned resources. - * - * @throws Exception If the audio device could not be closed. - */ - abstract public void close() throws Exception; - - /** - * Start capturing or playing audio by the device. - * - * @throws Exception If the device failed to start. - */ - abstract public void start() throws Exception; - - /** - * Stop capturing or playing audio by the device. - * - * @throws Exception If the device failed to stop. - */ - abstract public void stop() throws Exception; - - /** - * Check if the audio device is opened. - * - * @return {@code true} if the device is opened, otherwise {@code false}. - */ - abstract public boolean isOpen(); - - /** - * Get a list of all supported audio formats by this device. - * - * @return a list of all supported audio formats. - */ - abstract public List getSupportedFormats(); - - /** - * Get the current audio buffer size. The buffer size reflects the latency - * of the audio signal. - * - * @return the current audio buffer size. - */ - abstract public int getBufferSize(); - - - /** - * Check if the specified audio format is supported by the device. - * - * @param format The audio format to check. - * - * @return {@code true} if the audio format is supported, otherwise {@code false}. - */ - public boolean supportsAudioFormat(AudioFormat format) { - return getSupportedFormats().contains(format); - } - - /** - * Set the audio format to be used by the audio device for playback or - * recording. - * - * @param format The audio format to be used. - */ - public void setAudioFormat(AudioFormat format) { - this.audioFormat = format; - } - - /** - * Get the audio format of the device. - * - * @return the audio format. - */ - public AudioFormat getAudioFormat() { - return audioFormat; - } - - /** - * Get the volume of the device with which it plays or records audio. - * - * @return the volume of audio. - */ - public double getVolume() { - return volume; - } - - /** - * Set the audio volume for playback or recording. The volume value must be - * in the range of [0,1]. - * - * @param volume The new volume value. - */ - public void setVolume(double volume) { - if (volume < 0 || volume > 1) - return; - - this.volume = volume; - } - - /** - * Check whether audio signal is muted or not. - * - * @return {@code true} if the audio signal is muted, otherwise {@code false}. - */ - public boolean isMuted() { - return mute; - } - - /** - * Set whether to mute the audio signal of this device. - * - * @param mute True to mute the audio signal, false otherwise. - */ - public void setMute(boolean mute) { - this.mute = mute; - } - - /** - * Get the signal power level of the last processed audio data chunk. - * - * @return the current signal power level of audio. - */ - public double getSignalPowerLevel() { - if (isMuted()) { - return 0; - } - - return signalPowerLevel; - } - - /** - * Set the signal power level of the last processed audio data chunk. - * - * @param level the current signal power level of audio. - */ - protected void setSignalPowerLevel(float level) { - this.signalPowerLevel = level; - } - - /** - * AGC algorithm to adjust the speech level of an audio signal to a - * specified value in dBFS. - * - * @param input Audio input samples with values in range [-1, 1]. - * @param output Audio output samples with values in range [-1, 1]. - * @param gainLevel Output power level in dBFS. - * @param sampleCount Number of samples. - */ - protected void AGC(float[] input, float[] output, float gainLevel, int sampleCount) { - // Convert power gain level into normal power. - float power = (float) Math.pow(10, (gainLevel / 10)); - - // Calculate the energy of the input signal. - float energy = 0; - for (int i = 0; i < sampleCount; i++) { - energy += input[i] * input[i]; - } - - // Calculate the amplification factor. - float amp = (float) Math.sqrt((power * sampleCount) / energy); - - // Scale the input signal to achieve the required output power. - for (int i = 0; i < sampleCount; i++) { - output[i] = input[i] * amp; - } - } - - void applyGain(byte[] buffer, int offset, int length) { - if (volume == 1) { - signalPowerLevel = getSignalPowerLevel(buffer); - return; - } - - if (volume == 0) { - signalPowerLevel = 0; - Arrays.fill(buffer, offset, length - offset, (byte) 0); - return; - } - - float energy = 0; - - for (int i = 0; i < buffer.length; i += 2) { - int value = (short) ((buffer[i + 1] << 8) | (buffer[i] & 0xFF)); - value = (int) (volume * value); - - if (value > Short.MAX_VALUE) { - value = Short.MAX_VALUE; - } - else if (value < Short.MIN_VALUE) { - value = Short.MIN_VALUE; - } - - float norm = (float) value / Short.MAX_VALUE; - energy += norm * norm; - - buffer[i] = (byte) value; - buffer[i + 1] = (byte) (value >> 8); - } - - signalPowerLevel = (float) (10 * Math.log10(energy / (buffer.length / 2f)) + 96) / 96; - } - - private float getSignalPowerLevel(byte[] buffer) { - float energy = 0; - - for (int i = 0; i < buffer.length; i += 2) { - int value = (short) ((buffer[i + 1] << 8) | (buffer[i] & 0xFF)); - - float norm = (float) value / Short.MAX_VALUE; - energy += norm * norm; - } - - return (float) (10 * Math.log10(energy / (buffer.length / 2f)) + 96) / 96; + public String getName() { + return name; } } diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioInputDevice.java b/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioInputDevice.java deleted file mode 100644 index f2f4e664..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioInputDevice.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.device; - -/** - * Common class to provide a consistent mechanism for audio capture devices. - * - * @author Alex Andres - */ -public abstract class AudioInputDevice extends AudioDevice { - - /** - * Device specific method to be implemented to read captured audio data from - * the device into the specified audio buffer. - * - * @param buffer The audio buffer into which to write the captured audio. - * @param offset The offset at which to start to write the audio buffer. - * @param length The length of the audio buffer. - * - * @return the number of bytes written to the audio buffer. - * - * @throws Exception If captured audio could not be written to the buffer. - */ - abstract protected int readInput(byte[] buffer, int offset, int length) throws Exception; - - /** - * Read captured audio data from the device into the specified audio - * buffer. - * - * @param buffer The audio buffer into which to write the captured audio. - * @param offset The offset at which to start to write the audio buffer. - * @param length The length of the audio buffer. - * - * @return the number of bytes written to the audio buffer. - * - * @throws Exception If captured audio could not be written to the buffer. - */ - public synchronized int read(byte[] buffer, int offset, int length) throws Exception { - int read = readInput(buffer, offset, length); - - if (!isMuted()) { - applyGain(buffer, offset, length); - } - - return read; - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioOutputDevice.java b/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioOutputDevice.java deleted file mode 100644 index 0710bad0..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/device/AudioOutputDevice.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.device; - -/** - * Common class to provide a consistent mechanism for audio playback devices. - * - * @author Alex Andres - */ -public abstract class AudioOutputDevice extends AudioDevice { - - /** - * Device specific method to be implemented to write audio data for playback - * to the device from the specified audio buffer. - * - * @param buffer The audio buffer containing the samples for playback. - * @param offset The offset from which to start to read the audio buffer. - * @param length The length of the audio buffer. - * - * @return the number of bytes written to the playback buffer of the device. - * - * @throws Exception If the playback device did not accept the audio buffer. - */ - abstract public int writeOutput(byte[] buffer, int offset, int length) throws Exception; - - /** - * Write audio samples for playback. - * - * @param buffer The audio buffer containing the samples for playback. - * @param offset The offset from which to start to read the audio buffer. - * @param length The length of the audio buffer. - * - * @return the number of bytes written to the playback buffer of the device. - * - * @throws Exception If the playback device did not accept the audio buffer. - */ - public int write(byte[] buffer, int offset, int length) throws Exception { - int written = length; - - if (!isMuted()) { - applyGain(buffer, offset, length); - written = writeOutput(buffer, offset, length); - } - - return written; - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/device/FFmpegAudioInputDevice.java b/lect-core/src/main/java/org/lecturestudio/core/audio/device/FFmpegAudioInputDevice.java deleted file mode 100644 index ab64c8bd..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/device/FFmpegAudioInputDevice.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.device; - -import com.github.javaffmpeg.Audio; -import com.github.javaffmpeg.AudioFrame; - -import java.util.ArrayList; -import java.util.List; - -import org.lecturestudio.core.audio.AudioFormat; - -/** - * FFmpeg audio capture device implementation. - * - * @author Alex Andres - */ -public class FFmpegAudioInputDevice extends AudioInputDevice { - - /** Internal capture device. */ - private final com.github.javaffmpeg.FFmpegAudioInputDevice device; - - - /** - * Create a new {@link FFmpegAudioInputDevice} instance with the specified FFmpeg - * capture device. - * - * @param device The FFmpeg capture device. - */ - public FFmpegAudioInputDevice(com.github.javaffmpeg.FFmpegAudioInputDevice device) { - this.device = device; - } - - @Override - protected synchronized int readInput(byte[] buffer, int offset, int length) { - if (isOpen()) { - AudioFrame samples = device.getSamples(); - // TODO get by sample format - Audio.getAudio16(samples, buffer); - samples.clear(); - - return length; - } - - return 0; - } - - @Override - public String getName() { - return device.getName(); - } - - @Override - public synchronized void open() throws Exception { - if (isOpen()) { - return; - } - - AudioFormat audioFormat = getAudioFormat(); - - com.github.javaffmpeg.AudioFormat format = new com.github.javaffmpeg.AudioFormat(); - format.setChannelLayout(Audio.getChannelLayout(audioFormat.getChannels())); - format.setChannels(audioFormat.getChannels()); - format.setSampleFormat(Audio.getSampleFormat(audioFormat.getBytesPerSample(), false, false)); - format.setSampleRate(audioFormat.getSampleRate()); - - device.setBufferMilliseconds(20); - device.open(format); - } - - @Override - public synchronized void close() throws Exception { - device.close(); - } - - @Override - public synchronized void start() { - // nothing to do - } - - @Override - public synchronized void stop() { - // nothing to do - } - - @Override - public synchronized boolean isOpen() { - return device.isOpen(); - } - - @Override - public List getSupportedFormats() { - AudioFormat.Encoding encoding = AudioFormat.Encoding.S16LE; - int channels = 1; - - List formats = new ArrayList(); - - for (int sampleRate : AudioDevice.SUPPORTED_SAMPLE_RATES) { - AudioFormat audioFormat = new AudioFormat(encoding, sampleRate, channels); - - formats.add(audioFormat); - } - - return formats; - } - - @Override - public int getBufferSize() { - return device.getBufferSize(); - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/device/JavaSoundInputDevice.java b/lect-core/src/main/java/org/lecturestudio/core/audio/device/JavaSoundInputDevice.java deleted file mode 100644 index 08f2de62..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/device/JavaSoundInputDevice.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.device; - -import static java.util.Objects.isNull; -import static java.util.Objects.nonNull; - -import java.util.ArrayList; -import java.util.List; - -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.DataLine; -import javax.sound.sampled.Mixer; -import javax.sound.sampled.TargetDataLine; - -import org.lecturestudio.core.audio.AudioFormat; -import org.lecturestudio.core.util.OsInfo; - -/** - * Java-based audio capture device implementation. - * - * @author Alex Andres - */ -public class JavaSoundInputDevice extends AudioInputDevice { - - /** Minimal audio buffer size to use with Java. */ - private static final int BUFFER_SIZE = 4096; - - /** Internal {@link Mixer.Info} that represents information about an audio mixer. */ - private final Mixer.Info mixerInfo; - - /** Internal capture source. */ - private TargetDataLine line; - - - /** - * Create a new {@link JavaSoundInputDevice instance} with the specified {@link - * Mixer.Info} that contains information about an audio mixer. - * - * @param mixerInfo The audio mixer info. - */ - public JavaSoundInputDevice(Mixer.Info mixerInfo) { - this.mixerInfo = mixerInfo; - } - - @Override - public String getName() { - return mixerInfo.getName(); - } - - @Override - public void open() throws Exception { - if (isNull(mixerInfo)) { - throw new Exception("Invalid audio mixer set."); - } - - Mixer mixer = AudioSystem.getMixer(mixerInfo); - if (mixer == null) { - throw new Exception("Could not acquire specified mixer: " + getName()); - } - - AudioFormat audioFormat = getAudioFormat(); - javax.sound.sampled.AudioFormat format = createAudioFormat( - audioFormat.getSampleRate(), audioFormat.getChannels()); - - DataLine.Info info = new DataLine.Info(TargetDataLine.class, format); - - line = (TargetDataLine) mixer.getLine(info); - line.open(format, BUFFER_SIZE); - } - - @Override - public void close() throws Exception { - if (nonNull(line)) { - line.stop(); - line.flush(); - - if (!OsInfo.isMac()) { - line.close(); - } - - line = null; - } - } - - @Override - public void start() { - if (nonNull(line)) { - line.start(); - } - } - - @Override - public void stop() { - if (nonNull(line)) { - line.stop(); - line.flush(); - } - } - - @Override - public int readInput(byte[] buffer, int offset, int length) { - return line.read(buffer, offset, length); - } - - @Override - public List getSupportedFormats() { - AudioFormat.Encoding encoding = AudioFormat.Encoding.S16LE; - int channels = 1; - - List formats = new ArrayList<>(); - - Mixer mixer = AudioSystem.getMixer(mixerInfo); - DataLine.Info info; - - for (int sampleRate : AudioDevice.SUPPORTED_SAMPLE_RATES) { - AudioFormat audioFormat = new AudioFormat(encoding, sampleRate, channels); - javax.sound.sampled.AudioFormat format = createAudioFormat(sampleRate, channels); - - info = new DataLine.Info(TargetDataLine.class, format); - - if (mixer.isLineSupported(info)) { - formats.add(audioFormat); - } - } - - return formats; - } - - @Override - public int getBufferSize() { - if (nonNull(line)) { - return line.getBufferSize(); - } - - return -1; - } - - @Override - public boolean isOpen() { - return nonNull(line) && line.isOpen(); - } - - private javax.sound.sampled.AudioFormat createAudioFormat(int sampleRate, int channels) { - javax.sound.sampled.AudioFormat.Encoding encoding = javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED; - int sampleSizeInBits = 16; - int frameSize = (sampleSizeInBits / 8) * channels; - - return new javax.sound.sampled.AudioFormat( - encoding, sampleRate, sampleSizeInBits, channels, frameSize, - sampleRate, false); - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/device/JavaSoundOutputDevice.java b/lect-core/src/main/java/org/lecturestudio/core/audio/device/JavaSoundOutputDevice.java deleted file mode 100644 index b8927dd4..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/device/JavaSoundOutputDevice.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.device; - -import static java.util.Objects.isNull; -import static java.util.Objects.nonNull; - -import java.util.ArrayList; -import java.util.List; - -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.DataLine; -import javax.sound.sampled.Mixer; -import javax.sound.sampled.SourceDataLine; - -import org.lecturestudio.core.audio.AudioFormat; - -/** - * Java-based audio playback device implementation. - * - * @author Alex Andres - */ -public class JavaSoundOutputDevice extends AudioOutputDevice { - - /** Minimal audio buffer size to use with Java. */ - private static final int BUFFER_SIZE = 4096; - - /** Internal {@link Mixer.Info} that represents information about an audio mixer. */ - private final Mixer.Info mixerInfo; - - /** Internal playback sink. */ - private SourceDataLine line; - - - /** - * Create a new {@link JavaSoundOutputDevice} instance with the specified {@link - * Mixer.Info} that contains information about an audio mixer. - * - * @param mixerInfo The audio mixer info. - */ - public JavaSoundOutputDevice(Mixer.Info mixerInfo) { - this.mixerInfo = mixerInfo; - } - - @Override - public String getName() { - return mixerInfo.getName(); - } - - @Override - public void open() throws Exception { - if (isNull(mixerInfo)) { - throw new Exception("Invalid audio mixer set."); - } - - Mixer mixer = AudioSystem.getMixer(mixerInfo); - if (mixer == null) { - throw new Exception("Could not acquire specified mixer: " + getName()); - } - - AudioFormat audioFormat = getAudioFormat(); - javax.sound.sampled.AudioFormat format = createAudioFormat( - audioFormat.getSampleRate(), audioFormat.getChannels()); - - DataLine.Info info = new DataLine.Info(SourceDataLine.class, format); - - line = (SourceDataLine) mixer.getLine(info); - line.open(format, BUFFER_SIZE); - } - - @Override - public void close() throws Exception { - if (nonNull(line)) { - line.stop(); - line.flush(); - line.close(); - } - } - - @Override - public void start() { - if (nonNull(line)) { - line.start(); - } - } - - @Override - public void stop() { - if (nonNull(line)) { - line.stop(); - } - } - - @Override - public int writeOutput(byte[] buffer, int offset, int length) { - return line.write(buffer, offset, length); - } - - @Override - public List getSupportedFormats() { - AudioFormat.Encoding encoding = AudioFormat.Encoding.S16LE; - int[] supportedChannels = { 1, 2 }; - - List formats = new ArrayList<>(); - - Mixer mixer = AudioSystem.getMixer(mixerInfo); - DataLine.Info info; - - for (int channels : supportedChannels) { - for (int sampleRate : AudioDevice.SUPPORTED_SAMPLE_RATES) { - AudioFormat audioFormat = new AudioFormat(encoding, sampleRate, channels); - javax.sound.sampled.AudioFormat format = createAudioFormat(sampleRate, channels); - - info = new DataLine.Info(SourceDataLine.class, format); - - if (mixer.isLineSupported(info)) { - formats.add(audioFormat); - } - } - } - - return formats; - } - - @Override - public int getBufferSize() { - if (nonNull(line)) { - return line.getBufferSize(); - } - - return -1; - } - - @Override - public boolean isOpen() { - return nonNull(line) && line.isOpen(); - } - - private javax.sound.sampled.AudioFormat createAudioFormat(int sampleRate, int channels) { - javax.sound.sampled.AudioFormat.Encoding encoding = javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED; - int sampleSizeInBits = 16; - int frameSize = (sampleSizeInBits / 8) * channels; - - return new javax.sound.sampled.AudioFormat( - encoding, sampleRate, sampleSizeInBits, channels, frameSize, - sampleRate, false); - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/sink/AudioSink.java b/lect-core/src/main/java/org/lecturestudio/core/audio/sink/AudioSink.java index d8172da8..fee0b615 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/sink/AudioSink.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/sink/AudioSink.java @@ -70,9 +70,17 @@ public interface AudioSink { int write(byte[] data, int offset, int length) throws IOException; /** - * Set the audio format of audio samples the sink is ready to receive. + * Get the audio format of audio samples for this sink. * - * @param format The audio format of samples to receive. + * @return The audio format of samples to write. + */ + AudioFormat getAudioFormat(); + + /** + * Sets the {@code AudioFormat} of samples which will be provided to this + * {@code AudioSink}. + * + * @param format The audio format of audio samples. */ void setAudioFormat(AudioFormat format); diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/sink/ByteArrayAudioSink.java b/lect-core/src/main/java/org/lecturestudio/core/audio/sink/ByteArrayAudioSink.java new file mode 100644 index 00000000..c5a5682f --- /dev/null +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/sink/ByteArrayAudioSink.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.core.audio.sink; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.lecturestudio.core.audio.AudioFormat; + +/** + * AudioSink implementation which is backed by a {@code ByteArrayOutputStream}. + * + * @author Alex Andres + */ +public class ByteArrayAudioSink implements AudioSink { + + private ByteArrayOutputStream outputStream; + + private AudioFormat format; + + + @Override + public void open() throws IOException { + outputStream = new ByteArrayOutputStream(); + } + + @Override + public void reset() throws IOException { + outputStream.reset(); + } + + @Override + public void close() throws IOException { + outputStream.close(); + } + + @Override + public int write(byte[] data, int offset, int length) throws IOException { + try { + outputStream.write(data, 0, length); + } + catch (Exception e) { + throw new IOException(e); + } + + return length; + } + + @Override + public AudioFormat getAudioFormat() { + return format; + } + + @Override + public void setAudioFormat(AudioFormat format) { + this.format = format; + } + + public byte[] toByteArray() { + return outputStream.toByteArray(); + } +} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/sink/ProxyAudioSink.java b/lect-core/src/main/java/org/lecturestudio/core/audio/sink/ProxyAudioSink.java new file mode 100644 index 00000000..60d1ebcc --- /dev/null +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/sink/ProxyAudioSink.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.core.audio.sink; + +import java.io.IOException; + +import org.lecturestudio.core.audio.AudioFormat; + +/** + * AudioSink proxy implementation which redirects all calls to a specified + * sink. + * + * @author Alex Andres + */ +public class ProxyAudioSink implements AudioSink { + + private final AudioSink proxy; + + + public ProxyAudioSink(AudioSink proxy) { + this.proxy = proxy; + } + + @Override + public void open() throws IOException { + proxy.open(); + } + + @Override + public void reset() throws IOException { + proxy.reset(); + } + + @Override + public void close() throws IOException { + proxy.close(); + } + + @Override + public int write(byte[] data, int offset, int length) throws IOException { + return proxy.write(data, offset, length); + } + + @Override + public AudioFormat getAudioFormat() { + return proxy.getAudioFormat(); + } + + @Override + public void setAudioFormat(AudioFormat format) { + proxy.setAudioFormat(format); + } +} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/sink/WavFileSink.java b/lect-core/src/main/java/org/lecturestudio/core/audio/sink/WavFileSink.java index 017a9197..94ea2741 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/sink/WavFileSink.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/sink/WavFileSink.java @@ -88,6 +88,11 @@ public class WavFileSink implements AudioSink { return stream.write(data, offset, length); } + @Override + public AudioFormat getAudioFormat() { + return format; + } + @Override public void setAudioFormat(AudioFormat format) { this.format = format; diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/source/AudioInputStreamSource.java b/lect-core/src/main/java/org/lecturestudio/core/audio/source/AudioInputStreamSource.java index 8e5abe58..f112025e 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/source/AudioInputStreamSource.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/source/AudioInputStreamSource.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import org.lecturestudio.core.audio.AudioFormat; +import org.lecturestudio.core.audio.AudioUtils; /** * Audio input stream implementation of {@link AudioSource}. @@ -79,6 +80,17 @@ public class AudioInputStreamSource implements AudioSource { return audioStream.skip(n); } + @Override + public int seekMs(int timeMs) throws IOException { + float bytesPerSecond = AudioUtils.getBytesPerSecond(getAudioFormat()); + int skipBytes = Math.round(bytesPerSecond * timeMs / 1000F); + + reset(); + skip(skipBytes); + + return skipBytes; + } + @Override public AudioFormat getAudioFormat() { return audioFormat; diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/source/AudioSource.java b/lect-core/src/main/java/org/lecturestudio/core/audio/source/AudioSource.java index 30c11825..ab39bd5d 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/source/AudioSource.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/source/AudioSource.java @@ -70,6 +70,16 @@ public interface AudioSource { */ long skip(long n) throws IOException; + /** + * Jump to the specified time position in the audio playback stream. + * + * @param timeMs The absolute time in milliseconds to jump to. + * + * @throws IOException If the playback stream failed to read the start of + * the specified position. + */ + int seekMs(int timeMs) throws IOException; + /** * Get the number of bytes the audio source has available to read. * diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/source/ByteArrayAudioSource.java b/lect-core/src/main/java/org/lecturestudio/core/audio/source/ByteArrayAudioSource.java new file mode 100644 index 00000000..ed26c48f --- /dev/null +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/source/ByteArrayAudioSource.java @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.core.audio.source; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import org.lecturestudio.core.audio.AudioFormat; +import org.lecturestudio.core.audio.AudioUtils; + +public class ByteArrayAudioSource implements AudioSource { + + private final ByteArrayInputStream inputStream; + + private final AudioFormat format; + + + public ByteArrayAudioSource(ByteArrayInputStream inputStream, AudioFormat format) { + this.inputStream = inputStream; + this.format = format; + } + + @Override + public int read(byte[] data, int offset, int length) { + return inputStream.read(data, offset, length); + } + + @Override + public void close() throws IOException { + inputStream.close(); + } + + @Override + public void reset() { + inputStream.reset(); + } + + @Override + public long skip(long n) { + return inputStream.skip(n); + } + + @Override + public int seekMs(int timeMs) { + float bytesPerSecond = AudioUtils.getBytesPerSecond(format); + int skipBytes = Math.round(bytesPerSecond * timeMs / 1000F); + + reset(); + skip(skipBytes); + + return skipBytes; + } + + @Override + public long getInputSize() { + return inputStream.available(); + } + + @Override + public AudioFormat getAudioFormat() { + return format; + } +} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/source/RandomAccessAudioSource.java b/lect-core/src/main/java/org/lecturestudio/core/audio/source/RandomAccessAudioSource.java index 54acf4b3..d1ea52d9 100644 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/source/RandomAccessAudioSource.java +++ b/lect-core/src/main/java/org/lecturestudio/core/audio/source/RandomAccessAudioSource.java @@ -21,6 +21,7 @@ package org.lecturestudio.core.audio.source; import java.io.IOException; import org.lecturestudio.core.audio.AudioFormat; +import org.lecturestudio.core.audio.AudioUtils; import org.lecturestudio.core.io.RandomAccessAudioStream; import org.lecturestudio.core.model.Interval; @@ -50,6 +51,17 @@ public class RandomAccessAudioSource implements AudioSource { return stream.skip(n); } + @Override + public int seekMs(int timeMs) throws IOException { + float bytesPerSecond = AudioUtils.getBytesPerSecond(getAudioFormat()); + int skipBytes = Math.round(bytesPerSecond * timeMs / 1000F); + + reset(); + skip(skipBytes); + + return skipBytes; + } + @Override public void reset() throws IOException { stream.reset(); diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/system/AbstractSoundSystemProvider.java b/lect-core/src/main/java/org/lecturestudio/core/audio/system/AbstractSoundSystemProvider.java deleted file mode 100644 index 8acde081..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/system/AbstractSoundSystemProvider.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.system; - -import org.lecturestudio.core.audio.device.AudioInputDevice; -import org.lecturestudio.core.audio.device.AudioOutputDevice; - -/** - * Base {@link AudioSystemProvider} implementation that implements the default retrieval - * of audio devices. - * - * @author Alex Andres - */ -public abstract class AbstractSoundSystemProvider implements AudioSystemProvider { - - @Override - public AudioInputDevice getInputDevice(String deviceName) { - for (AudioInputDevice device : getInputDevices()) { - if (device.getName().equals(deviceName)) { - return device; - } - } - return null; - } - - @Override - public AudioOutputDevice getOutputDevice(String deviceName) { - for (AudioOutputDevice device : getOutputDevices()) { - if (device.getName().equals(deviceName)) { - return device; - } - } - return null; - } - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/system/AudioSystemLoader.java b/lect-core/src/main/java/org/lecturestudio/core/audio/system/AudioSystemLoader.java deleted file mode 100644 index cd46725a..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/system/AudioSystemLoader.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.system; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.lecturestudio.core.spi.SpiLoader; - -/** - * SPI audio system loader. To provide different audio system implementations - * the system provider must implement the {@link AudioSystemProvider} interface - * and be registered via the SPI. - * - * @author Alex Andres - * @see - * https://docs.oracle.com/javase/tutorial/ext/basics/spi.html - */ -public class AudioSystemLoader extends SpiLoader { - - /** The audio system loader. */ - private static AudioSystemLoader serviceLoader; - - /** The Java-based sound system provided. */ - private final AudioSystemProvider javaSoundProvider = new JavaSoundProvider(); - - - /** - * Retrieve the singleton instance of {@link AudioSystemLoader}. - */ - public static synchronized AudioSystemLoader getInstance() { - if (serviceLoader == null) { - serviceLoader = new AudioSystemLoader(); - } - return serviceLoader; - } - - /** - * Get an {@link AudioSystemProvider} with the specified name. - * - * @param providerName The name of the {@link AudioSystemProvider} to retrieve. - * - * @return the {@link AudioSystemProvider} or null, if no such provider exists. - */ - public AudioSystemProvider getProvider(String providerName) { - if (javaSoundProvider.getProviderName().equals(providerName)) { - return javaSoundProvider; - } - - return super.getProvider(providerName); - } - - /** - * Retrieve the names of all registered audio system providers. - * - * @return an array of names of all registered audio system providers. - */ - public String[] getProviderNames() { - List names = new ArrayList<>(); - Collections.addAll(names, super.getProviderNames()); - names.add(javaSoundProvider.getProviderName()); - - return names.toArray(new String[0]); - } - - private AudioSystemLoader() { - super(new File[] { new File("lib"), new File("../../lib") }, - "org.lecturestudio.core.audio.system", AudioSystemProvider.class); - } -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/system/AudioSystemProvider.java b/lect-core/src/main/java/org/lecturestudio/core/audio/system/AudioSystemProvider.java deleted file mode 100644 index 4a2d16d0..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/system/AudioSystemProvider.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.system; - -import org.lecturestudio.core.audio.device.AudioInputDevice; -import org.lecturestudio.core.audio.device.AudioOutputDevice; -import org.lecturestudio.core.spi.ServiceProvider; - -/** - * Audio system provider implementation. The {@link AudioSystemProvider} provides access - * implementation specific audio devices. - * - * @author Alex Andres - */ -public interface AudioSystemProvider extends ServiceProvider { - - /** - * Get the systems default audio capture device. - * - * @return The default audio capture device. - */ - AudioInputDevice getDefaultInputDevice(); - - /** - * Get the systems default audio playback device. - * - * @return The default audio playback device. - */ - AudioOutputDevice getDefaultOutputDevice(); - - /** - * Get all available audio capture devices. - * - * @return an array of all audio capture devices. - */ - AudioInputDevice[] getInputDevices(); - - /** - * Get all available audio playback devices. - * - * @return an array of all audio playback devices. - */ - AudioOutputDevice[] getOutputDevices(); - - /** - * Get an audio capture device with the specified device name. - * - * @param deviceName The name of the device to retrieve. - * - * @return an audio capture device. - */ - AudioInputDevice getInputDevice(String deviceName); - - /** - * Get an audio playback device with the specified device name. - * - * @param deviceName The name of the device to retrieve. - * - * @return an audio playback device. - */ - AudioOutputDevice getOutputDevice(String deviceName); - -} diff --git a/lect-core/src/main/java/org/lecturestudio/core/audio/system/JavaSoundProvider.java b/lect-core/src/main/java/org/lecturestudio/core/audio/system/JavaSoundProvider.java deleted file mode 100644 index 6325446d..00000000 --- a/lect-core/src/main/java/org/lecturestudio/core/audio/system/JavaSoundProvider.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (C) 2020 TU Darmstadt, Department of Computer Science, - * Embedded Systems and Applications Group. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.lecturestudio.core.audio.system; - -import static java.util.Objects.isNull; - -import java.util.ArrayList; -import java.util.List; - -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.Line; -import javax.sound.sampled.Mixer; -import javax.sound.sampled.SourceDataLine; -import javax.sound.sampled.TargetDataLine; - -import org.lecturestudio.core.audio.device.AudioInputDevice; -import org.lecturestudio.core.audio.device.AudioOutputDevice; -import org.lecturestudio.core.audio.device.JavaSoundInputDevice; -import org.lecturestudio.core.audio.device.JavaSoundOutputDevice; - -/** - * Java-based sound system provider implementation. - * - * @author Alex Andres - */ -public class JavaSoundProvider extends AbstractSoundSystemProvider { - - @Override - public AudioInputDevice getDefaultInputDevice() { - AudioInputDevice[] devices = getInputDevices(); - - return (isNull(devices) || devices.length < 1) ? null : devices[0]; - } - - @Override - public AudioOutputDevice getDefaultOutputDevice() { - AudioOutputDevice[] devices = getOutputDevices(); - - return (isNull(devices) || devices.length < 1) ? null : devices[0]; - } - - @Override - public AudioInputDevice[] getInputDevices() { - List inputDevices = new ArrayList<>(); - Mixer.Info[] mixerInfo = AudioSystem.getMixerInfo(); - - for (Mixer.Info info : mixerInfo) { - Mixer mixer = AudioSystem.getMixer(info); - Line.Info targetInfo = new Line.Info(TargetDataLine.class); - - if (mixer.isLineSupported(targetInfo)) { - JavaSoundInputDevice device = new JavaSoundInputDevice(info); - inputDevices.add(device); - } - } - - return inputDevices.toArray(new AudioInputDevice[0]); - } - - @Override - public AudioOutputDevice[] getOutputDevices() { - List outputDevices = new ArrayList<>(); - Mixer.Info[] mixerInfo = AudioSystem.getMixerInfo(); - - for (Mixer.Info info : mixerInfo) { - Mixer mixer = AudioSystem.getMixer(info); - Line.Info sourceInfo = new Line.Info(SourceDataLine.class); - - if (mixer.isLineSupported(sourceInfo)) { - JavaSoundOutputDevice device = new JavaSoundOutputDevice(info); - outputDevices.add(device); - } - } - - return outputDevices.toArray(new AudioOutputDevice[0]); - } - - @Override - public String getProviderName() { - return "Java Sound"; - } - -} diff --git a/lect-core/src/main/resources/META-INF/services/org.lecturestudio.core.audio.codec.AudioCodecProvider b/lect-core/src/main/resources/META-INF/services/org.lecturestudio.core.audio.codec.AudioCodecProvider deleted file mode 100644 index d5fcc10a..00000000 --- a/lect-core/src/main/resources/META-INF/services/org.lecturestudio.core.audio.codec.AudioCodecProvider +++ /dev/null @@ -1 +0,0 @@ -org.lecturestudio.core.audio.codec.OpusCodecProvider \ No newline at end of file diff --git a/lect-core/src/test/java/org/lecturestudio/core/app/configuration/ConfigurationTest.java b/lect-core/src/test/java/org/lecturestudio/core/app/configuration/ConfigurationTest.java index dca0200c..2c426bab 100644 --- a/lect-core/src/test/java/org/lecturestudio/core/app/configuration/ConfigurationTest.java +++ b/lect-core/src/test/java/org/lecturestudio/core/app/configuration/ConfigurationTest.java @@ -199,7 +199,6 @@ class ConfigurationTest { config.getAudioConfig().setPlaybackDeviceName("Speakers"); config.getAudioConfig().setRecordingFormat(new AudioFormat(AudioFormat.Encoding.S16LE, 44100, 1)); config.getAudioConfig().setRecordingPath("/home/tmp"); - config.getAudioConfig().setSoundSystem("Java"); config.getAudioConfig().setRecordingVolume("Microphone", 0.7f); manager.save(configFile, config); @@ -211,7 +210,6 @@ class ConfigurationTest { assertEquals("Speakers", audioConfig.getPlaybackDeviceName()); assertEquals(audioConfig.getRecordingFormat(), new AudioFormat(AudioFormat.Encoding.S16LE, 44100, 1)); assertEquals("/home/tmp", audioConfig.getRecordingPath()); - assertEquals("Java", audioConfig.getSoundSystem()); assertEquals(Double.valueOf(0.7f), audioConfig.getRecordingVolume("Microphone")); } diff --git a/lect-editor-api/src/main/java/org/lecturestudio/editor/api/config/DefaultConfiguration.java b/lect-editor-api/src/main/java/org/lecturestudio/editor/api/config/DefaultConfiguration.java index f8930ec9..f66cad9d 100644 --- a/lect-editor-api/src/main/java/org/lecturestudio/editor/api/config/DefaultConfiguration.java +++ b/lect-editor-api/src/main/java/org/lecturestudio/editor/api/config/DefaultConfiguration.java @@ -81,7 +81,6 @@ public class DefaultConfiguration extends EditorConfiguration { getToolConfig().getPresetColors().addAll(new ArrayList<>(6)); - getAudioConfig().setSoundSystem("Java Sound"); getAudioConfig().setPlaybackVolume(1); } diff --git a/lect-editor-api/src/main/java/org/lecturestudio/editor/api/presenter/NoiseReductionSettingsPresenter.java b/lect-editor-api/src/main/java/org/lecturestudio/editor/api/presenter/NoiseReductionSettingsPresenter.java index c1de6efe..9eea9234 100644 --- a/lect-editor-api/src/main/java/org/lecturestudio/editor/api/presenter/NoiseReductionSettingsPresenter.java +++ b/lect-editor-api/src/main/java/org/lecturestudio/editor/api/presenter/NoiseReductionSettingsPresenter.java @@ -35,11 +35,6 @@ import org.lecturestudio.core.ExecutableState; import org.lecturestudio.core.app.ApplicationContext; import org.lecturestudio.core.app.configuration.Configuration; import org.lecturestudio.core.app.dictionary.Dictionary; -import org.lecturestudio.core.audio.AudioPlayer; -import org.lecturestudio.core.audio.AudioUtils; -import org.lecturestudio.core.audio.Player; -import org.lecturestudio.core.audio.SyncState; -import org.lecturestudio.core.audio.device.AudioOutputDevice; import org.lecturestudio.core.audio.effect.DenoiseEffectRunner; import org.lecturestudio.core.audio.effect.NoiseReductionParameters; import org.lecturestudio.core.audio.sink.AudioSink; @@ -57,6 +52,7 @@ import org.lecturestudio.core.presenter.Presenter; import org.lecturestudio.core.recording.Recording; import org.lecturestudio.core.view.NotificationType; import org.lecturestudio.editor.api.context.EditorContext; +import org.lecturestudio.media.webrtc.WebRtcAudioPlayer; import org.lecturestudio.media.recording.RecordingEvent; import org.lecturestudio.editor.api.presenter.command.NoiseReductionProgressCommand; import org.lecturestudio.editor.api.service.RecordingFileService; @@ -77,7 +73,7 @@ public class NoiseReductionSettingsPresenter extends Presenter. + */ + +package org.lecturestudio.editor.api.presenter; + +import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.lecturestudio.core.app.ApplicationContext; +import org.lecturestudio.core.app.configuration.AudioConfiguration; +import org.lecturestudio.core.app.configuration.Configuration; +import org.lecturestudio.core.audio.AudioSystemProvider; +import org.lecturestudio.core.audio.device.AudioDevice; +import org.lecturestudio.core.presenter.Presenter; +import org.lecturestudio.editor.api.config.DefaultConfiguration; +import org.lecturestudio.editor.api.view.SoundSettingsView; + +public class SoundSettingsPresenter extends Presenter { + + private final AudioConfiguration audioConfig; + + private final AudioSystemProvider audioSystemProvider; + + + @Inject + SoundSettingsPresenter(ApplicationContext context, SoundSettingsView view, + AudioSystemProvider audioSystemProvider) { + super(context, view); + + this.audioConfig = context.getConfiguration().getAudioConfig(); + this.audioSystemProvider = audioSystemProvider; + } + + @Override + public void initialize() throws Exception { + if (isNull(audioConfig.getPlaybackDeviceName())) { + setDefaultPlaybackDevice(); + } + + var devices = Arrays.stream(audioSystemProvider.getPlaybackDevices()) + .map(AudioDevice::getName).collect(Collectors.toList()); + + view.setAudioPlaybackDevices(devices); + view.setAudioPlaybackDevice(audioConfig.playbackDeviceNameProperty()); + + view.setOnClose(this::close); + view.setOnReset(this::reset); + } + + private void reset() { + Configuration config = context.getConfiguration(); + DefaultConfiguration defaultConfig = new DefaultConfiguration(); + + config.getAudioConfig().setPlaybackDeviceName( + defaultConfig.getAudioConfig().getPlaybackDeviceName()); + } + + private void setDefaultPlaybackDevice() { + AudioDevice playbackDevice = audioSystemProvider.getDefaultPlaybackDevice(); + + // Select first available playback device. + if (nonNull(playbackDevice)) { + audioConfig.setPlaybackDeviceName(playbackDevice.getName()); + } + } +} \ No newline at end of file diff --git a/lect-editor-api/src/main/java/org/lecturestudio/editor/api/service/RecordingPlaybackService.java b/lect-editor-api/src/main/java/org/lecturestudio/editor/api/service/RecordingPlaybackService.java index f88fe283..1356bf44 100644 --- a/lect-editor-api/src/main/java/org/lecturestudio/editor/api/service/RecordingPlaybackService.java +++ b/lect-editor-api/src/main/java/org/lecturestudio/editor/api/service/RecordingPlaybackService.java @@ -32,6 +32,7 @@ import org.lecturestudio.core.ExecutableState; import org.lecturestudio.core.ExecutableStateListener; import org.lecturestudio.core.app.ApplicationContext; import org.lecturestudio.core.app.configuration.AudioConfiguration; +import org.lecturestudio.core.audio.AudioSystemProvider; import org.lecturestudio.core.audio.filter.AudioFilter; import org.lecturestudio.core.model.Interval; import org.lecturestudio.core.model.Page; @@ -45,6 +46,8 @@ public class RecordingPlaybackService extends ExecutableBase { private final static Logger LOG = LogManager.getLogger(RecordingPlaybackService.class); + private final AudioSystemProvider audioSystemProvider; + private final EditorContext context; private final ExecutableStateListener playbackStateListener = (oldState, newState) -> { @@ -62,7 +65,8 @@ public class RecordingPlaybackService extends ExecutableBase { @Inject - RecordingPlaybackService(ApplicationContext context) { + RecordingPlaybackService(ApplicationContext context, AudioSystemProvider audioSystemProvider) { + this.audioSystemProvider = audioSystemProvider; this.context = (EditorContext) context; this.context.primarySelectionProperty().addListener((o, oldValue, newValue) -> { if (initialized() || suspended()) { @@ -93,7 +97,9 @@ public class RecordingPlaybackService extends ExecutableBase { closeRecording(); } - recordingPlayer = new RecordingPlayer(context, context.getConfiguration().getAudioConfig()); + recordingPlayer = new RecordingPlayer(context, + context.getConfiguration().getAudioConfig(), + audioSystemProvider); recordingPlayer.setRecording(recording); try { diff --git a/lect-editor-api/src/main/java/org/lecturestudio/editor/api/view/SoundSettingsView.java b/lect-editor-api/src/main/java/org/lecturestudio/editor/api/view/SoundSettingsView.java new file mode 100644 index 00000000..e1e8f972 --- /dev/null +++ b/lect-editor-api/src/main/java/org/lecturestudio/editor/api/view/SoundSettingsView.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.editor.api.view; + +import java.util.List; + +import org.lecturestudio.core.beans.StringProperty; + +public interface SoundSettingsView extends SettingsBaseView { + + void setAudioPlaybackDevice(StringProperty deviceName); + + void setAudioPlaybackDevices(List devices); + +} diff --git a/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/EditorFxApplication.java b/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/EditorFxApplication.java index a73901ec..c50f1892 100644 --- a/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/EditorFxApplication.java +++ b/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/EditorFxApplication.java @@ -18,6 +18,8 @@ package org.lecturestudio.editor.javafx; +import dev.onvoid.webrtc.logging.Logging; + import org.lecturestudio.core.app.ApplicationFactory; import org.lecturestudio.javafx.app.JavaFxApplication; @@ -30,6 +32,8 @@ public class EditorFxApplication extends JavaFxApplication { * @param args the main method's arguments. */ public static void main(String[] args) { + Logging.logThreads(true); + // Start with pre-loader. EditorFxApplication.launch(args, EditorFxPreloader.class); } diff --git a/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/inject/guice/ApplicationModule.java b/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/inject/guice/ApplicationModule.java index 9ad13cfa..423a9d3c 100644 --- a/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/inject/guice/ApplicationModule.java +++ b/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/inject/guice/ApplicationModule.java @@ -36,6 +36,7 @@ import org.lecturestudio.core.app.configuration.Configuration; import org.lecturestudio.core.app.configuration.ConfigurationService; import org.lecturestudio.core.app.configuration.JsonConfigurationService; import org.lecturestudio.core.app.dictionary.Dictionary; +import org.lecturestudio.core.audio.AudioSystemProvider; import org.lecturestudio.core.audio.bus.AudioBus; import org.lecturestudio.core.bus.ApplicationBus; import org.lecturestudio.core.bus.EventBus; @@ -49,6 +50,7 @@ import org.lecturestudio.core.util.DirUtils; import org.lecturestudio.editor.api.config.DefaultConfiguration; import org.lecturestudio.editor.api.config.EditorConfiguration; import org.lecturestudio.editor.api.context.EditorContext; +import org.lecturestudio.media.webrtc.WebRtcAudioSystemProvider; import org.lecturestudio.swing.DefaultRenderContext; import org.apache.logging.log4j.LogManager; @@ -66,6 +68,7 @@ public class ApplicationModule extends AbstractModule { @Override protected void configure() { bind(ToolController.class).asEagerSingleton(); + bind(AudioSystemProvider.class).to(WebRtcAudioSystemProvider.class); } @Provides diff --git a/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/inject/guice/ViewModule.java b/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/inject/guice/ViewModule.java index 5282c9ab..8e4039a3 100644 --- a/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/inject/guice/ViewModule.java +++ b/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/inject/guice/ViewModule.java @@ -67,6 +67,7 @@ public class ViewModule extends AbstractModule { bind(ReplacePageView.class).to(FxReplacePageView.class); bind(SettingsView.class).to(FxSettingsView.class); bind(SlidesView.class).to(FxSlidesView.class); + bind(SoundSettingsView.class).to(FxSoundSettingsView.class); bind(StartView.class).to(FxStartView.class); bind(VideoExportView.class).to(FxVideoExportView.class); bind(VideoExportProgressView.class).to(FxVideoExportProgressView.class); diff --git a/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/view/FxSoundSettingsView.java b/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/view/FxSoundSettingsView.java new file mode 100644 index 00000000..dcdb8ed4 --- /dev/null +++ b/lect-editor-fx/src/main/java/org/lecturestudio/editor/javafx/view/FxSoundSettingsView.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2021 TU Darmstadt, Department of Computer Science, + * Embedded Systems and Applications Group. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.lecturestudio.editor.javafx.view; + +import java.util.List; + +import org.lecturestudio.core.beans.StringProperty; +import org.lecturestudio.core.view.Action; +import org.lecturestudio.editor.api.view.SoundSettingsView; +import org.lecturestudio.javafx.beans.LectStringProperty; +import org.lecturestudio.javafx.util.FxUtils; +import org.lecturestudio.javafx.view.FxmlView; + +import javafx.fxml.FXML; +import javafx.scene.control.Button; +import javafx.scene.control.ComboBox; +import javafx.scene.layout.GridPane; + +@FxmlView(name = "sound-settings", presenter = org.lecturestudio.editor.api.presenter.SoundSettingsPresenter.class) +public class FxSoundSettingsView extends GridPane implements SoundSettingsView { + + @FXML + private ComboBox playbackDeviceCombo; + + @FXML + private Button closeButton; + + @FXML + private Button resetButton; + + + public FxSoundSettingsView() { + super(); + } + + @Override + public void setAudioPlaybackDevice(StringProperty deviceName) { + playbackDeviceCombo.valueProperty() + .bindBidirectional(new LectStringProperty(deviceName)); + } + + @Override + public void setAudioPlaybackDevices(List devices) { + FxUtils.invoke(() -> playbackDeviceCombo.getItems().setAll(devices)); + } + + @Override + public void setOnClose(Action action) { + FxUtils.bindAction(closeButton, action); + } + + @Override + public void setOnReset(Action action) { + FxUtils.bindAction(resetButton, action); + } +} diff --git a/lect-editor-fx/src/main/resources/views/general-settings/general-settings.css b/lect-editor-fx/src/main/resources/views/general-settings/general-settings.css new file mode 100644 index 00000000..71f29f7a --- /dev/null +++ b/lect-editor-fx/src/main/resources/views/general-settings/general-settings.css @@ -0,0 +1,11 @@ +.general-settings { + -fx-hgap: 10; + -fx-vgap: 10; + -fx-padding: 2em; +} +.general-settings > .label { + -fx-padding: 0 0 0.425em 0; +} +.general-settings > .text-small { + -fx-padding: 0.15em 0 0 0; +} \ No newline at end of file diff --git a/lect-editor-fx/src/main/resources/views/general-settings/general-settings.fxml b/lect-editor-fx/src/main/resources/views/general-settings/general-settings.fxml index 6d8e930b..6c56d333 100644 --- a/lect-editor-fx/src/main/resources/views/general-settings/general-settings.fxml +++ b/lect-editor-fx/src/main/resources/views/general-settings/general-settings.fxml @@ -1,11 +1,10 @@ - - + @@ -39,8 +38,4 @@ - + - + diff --git a/lect-presenter-swing/src/main/resources/views/sound-settings/sound-settings_de_DE.properties b/lect-presenter-swing/src/main/resources/views/sound-settings/sound-settings_de_DE.properties index ada17552..1f85fd35 100644 --- a/lect-presenter-swing/src/main/resources/views/sound-settings/sound-settings_de_DE.properties +++ b/lect-presenter-swing/src/main/resources/views/sound-settings/sound-settings_de_DE.properties @@ -3,6 +3,11 @@ sound.settings.capture.device = Eingabeger\u00e4t sound.settings.playback.device = Ausgabeger\u00e4t sound.settings.level = Pegel sound.settings.volume = Lautst\u00e4rke +sound.settings.noise.suppression = Rauschunterdr\u00fcckung +sound.settings.noise.suppression.moderate = Moderat +sound.settings.noise.suppression.low = Niedrig +sound.settings.noise.suppression.high = Hoch +sound.settings.noise.suppression.very_high = Sehr hoch sound.settings.test.title = Aufnahme Testen sound.settings.test.capture = Aufnahme sound.settings.test.playback = Wiedergabe diff --git a/lect-presenter-swing/src/main/resources/views/sound-settings/sound-settings_en_US.properties b/lect-presenter-swing/src/main/resources/views/sound-settings/sound-settings_en_US.properties index 7e6f8b40..b1706fbd 100644 --- a/lect-presenter-swing/src/main/resources/views/sound-settings/sound-settings_en_US.properties +++ b/lect-presenter-swing/src/main/resources/views/sound-settings/sound-settings_en_US.properties @@ -3,6 +3,11 @@ sound.settings.capture.device = Input device sound.settings.playback.device = Output device sound.settings.level = Level sound.settings.volume = Volume +sound.settings.noise.suppression = Noise suppression +sound.settings.noise.suppression.moderate = Moderate +sound.settings.noise.suppression.low = Low +sound.settings.noise.suppression.high = High +sound.settings.noise.suppression.very_high = Very high sound.settings.test.title = Test Capture sound.settings.test.capture = Capture sound.settings.test.playback = Playback diff --git a/lect-web-parent/lect-web-api/pom.xml b/lect-web-parent/lect-web-api/pom.xml index df5d54d7..ca320ea9 100644 --- a/lect-web-parent/lect-web-api/pom.xml +++ b/lect-web-parent/lect-web-api/pom.xml @@ -43,7 +43,7 @@ dev.onvoid.webrtc webrtc-java - 0.4.0 + 0.5.0 diff --git a/lib/native/linux-x86_64/libavdev.so b/lib/native/linux-x86_64/libavdev.so deleted file mode 100644 index d7733c4d..00000000 Binary files a/lib/native/linux-x86_64/libavdev.so and /dev/null differ diff --git a/lib/native/macosx-x86_64/libavdev.dylib b/lib/native/macosx-x86_64/libavdev.dylib deleted file mode 100644 index 4eae5ef4..00000000 Binary files a/lib/native/macosx-x86_64/libavdev.dylib and /dev/null differ diff --git a/lib/native/windows-x86_64/avdev.dll b/lib/native/windows-x86_64/avdev.dll deleted file mode 100644 index f741ea70..00000000 Binary files a/lib/native/windows-x86_64/avdev.dll and /dev/null differ