diff --git a/app/meson.build b/app/meson.build index d3336daae7..4665ba9aef 100644 --- a/app/meson.build +++ b/app/meson.build @@ -105,6 +105,12 @@ if v4l2_support src += [ 'src/v4l2_sink.c' ] endif +libavdevice_dep = dependency('libavdevice', static: get_option('static'), required: false) +client_audio_support = get_option('client_audio') and libavdevice_dep.found() +if client_audio_support + src += [ 'src/client_audio.c' ] +endif + usb_support = get_option('usb') if usb_support src += [ @@ -129,8 +135,8 @@ dependencies = [ dependency('sdl3', version: '>= 3.2.0', static: static), ] -if v4l2_support - dependencies += dependency('libavdevice', static: static) +if v4l2_support or client_audio_support + dependencies += libavdevice_dep endif if usb_support @@ -182,6 +188,9 @@ conf.set('SERVER_DEBUGGER', get_option('server_debugger')) # enable V4L2 support (linux only) conf.set('HAVE_V4L2', v4l2_support) +# enable client audio forwarding (requires libavdevice) +conf.set('HAVE_CLIENT_AUDIO', client_audio_support) + # enable HID over AOA support (linux only) conf.set('HAVE_USB', usb_support) diff --git a/app/src/cli.c b/app/src/cli.c index 39576d8d20..949a2f1291 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -41,6 +41,7 @@ enum { OPT_SHORTCUT_MOD, OPT_NO_KEY_REPEAT, OPT_LEGACY_PASTE, + OPT_CLIENT_AUDIO_SOURCE, OPT_VIDEO_ENCODER, OPT_POWER_OFF_ON_CLOSE, OPT_V4L2_SINK, @@ -106,6 +107,7 @@ enum { OPT_MIN_SIZE_ALIGNMENT, OPT_NO_WINDOW_ASPECT_RATIO_LOCK, OPT_KEEP_ACTIVE, + OPT_LIST_AUDIO_SOURCES, }; struct sc_option { @@ -384,6 +386,16 @@ static const struct sc_option options[] = { .text = "Use TCP/IP device (if there is exactly one, like adb -e).\n" "Also see -d (--select-usb).", }, + { + .longopt_id = OPT_CLIENT_AUDIO_SOURCE, + .longopt = "client-audio-source", + .argdesc = "source", + .text = "Inject audio into the device microphone from a device or file.\n" + "The source can be:\n" + " - A device name (e.g., \"Microphone\", \"default\")\n" + " - A file path prefixed with \"file://\" (e.g., \"file:///path/to/audio.mp3\")\n" + "Supported file formats: MP3, OGG, WAV, FLAC, etc.", + }, { .shortopt = 'f', .longopt = "fullscreen", @@ -488,6 +500,11 @@ static const struct sc_option options[] = { .longopt = "list-encoders", .text = "List video and audio encoders available on the device.", }, + { + .longopt_id = OPT_LIST_AUDIO_SOURCES, + .longopt = "list-client-audio-sources", + .text = "List available audio input sources on the client computer.", + }, { .shortopt = 'm', .longopt = "max-size", @@ -2547,6 +2564,14 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_AUDIO_CODEC_OPTIONS: opts->audio_codec_options = optarg; break; + case OPT_CLIENT_AUDIO_SOURCE: +#ifdef HAVE_CLIENT_AUDIO + opts->client_audio_source = optarg; + break; +#else + LOGE("Client audio (--client-audio-source) is disabled in this build"); + return false; +#endif case OPT_VIDEO_ENCODER: opts->video_encoder = optarg; break; @@ -2650,6 +2675,14 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_LIST_APPS: opts->list |= SC_OPTION_LIST_APPS; break; + case OPT_LIST_AUDIO_SOURCES: +#ifdef HAVE_CLIENT_AUDIO + args->list_audio_sources = true; + break; +#else + LOGE("Client audio (--list-client-audio-sources) is disabled in this build"); + return false; +#endif case OPT_REQUIRE_AUDIO: opts->require_audio = true; break; @@ -2833,7 +2866,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], opts->audio = false; } - if (!opts->video && !opts->audio && !opts->control && !otg) { + bool has_client_audio = false; +#ifdef HAVE_CLIENT_AUDIO + has_client_audio = opts->client_audio_source != NULL; +#endif + if (!opts->video && !opts->audio && !opts->control && !otg && !has_client_audio) { LOGE("No video, no audio, no control, no OTG: nothing to do"); return false; } @@ -2947,6 +2984,15 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } } +#ifdef HAVE_CLIENT_AUDIO + if (opts->client_audio_source && + (opts->audio_source == SC_AUDIO_SOURCE_VOICE_CALL || + opts->audio_source == SC_AUDIO_SOURCE_VOICE_CALL_UPLINK)) { + LOGE("--client-audio-source is incompatible with --audio-source=voice-call and --audio-source=voice-call-uplink"); + return false; + } +#endif + if (otg) { if (!opts->control) { LOGE("--no-control is not allowed in OTG mode"); diff --git a/app/src/cli.h b/app/src/cli.h index 9aa38dc618..5499340e44 100644 --- a/app/src/cli.h +++ b/app/src/cli.h @@ -18,6 +18,7 @@ struct scrcpy_cli_args { struct scrcpy_options opts; bool help; bool version; + bool list_audio_sources; enum sc_pause_on_exit pause_on_exit; }; diff --git a/app/src/client_audio.c b/app/src/client_audio.c new file mode 100644 index 0000000000..3030eee3a0 --- /dev/null +++ b/app/src/client_audio.c @@ -0,0 +1,513 @@ +#include "client_audio.h" +#include "util/binary.h" +#include "util/log.h" +#include "util/net.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Detect platform-specific audio input format +static const char *detect_audio_format(void) { +#ifdef _WIN32 + // Try WASAPI first (more modern), fallback to dshow + if (av_find_input_format("wasapi")) { + return "wasapi"; + } + return "dshow"; +#elif defined(__APPLE__) + return "avfoundation"; +#else + // Linux and others + return "alsa"; +#endif +} + +// Parse audio source to determine if it's a file or device +// Returns true if it's a file, false if it's a device +static bool parse_audio_source(const char *source, const char **path) { + if (strncmp(source, "file://", 7) == 0) { + *path = source + 7; // Skip "file://" + return true; + } + *path = source; + return false; +} + +// List available audio input sources on the client machine +void +sc_microphone_list_audio_sources(void) { + const char *format_name = detect_audio_format(); + AVInputFormat *input_format = av_find_input_format(format_name); + + if (!input_format) { + LOGE("Could not find audio input format '%s'", format_name); + return; + } + + LOGI("Audio input format: %s (%s)", + input_format->name, + input_format->long_name ? input_format->long_name : "no description"); + +#ifdef __APPLE__ + LOGI("Listing devices via avfoundation (see output above/below):"); + + AVDictionary *options = NULL; + av_dict_set(&options, "list_devices", "true", 0); + + AVFormatContext *fmt_ctx = NULL; + avformat_open_input(&fmt_ctx, "", input_format, &options); + + av_dict_free(&options); + if (fmt_ctx) { + avformat_close_input(&fmt_ctx); + } + + LOGI("How to use:"); + LOGI(" Use the audio device index with --client-audio-source"); + LOGI(" Format: \":AUDIO_INDEX\" (e.g., \":0\" for first audio device)"); + LOGI("Examples:"); + LOGI(" scrcpy --client-audio-source :0"); + LOGI(" scrcpy --client-audio-source file:///path/to/audio.mp3"); + return; +#endif + + LOGI("Available audio sources:"); + + AVDeviceInfoList *device_list = NULL; + int ret = avdevice_list_input_sources(input_format, NULL, NULL, &device_list); + + if (ret < 0) { + LOGE("Could not list audio devices (error code: %d)", ret); + LOGI("Note: You can still use device names directly with --client-audio-source"); + LOGI("Common device names:"); +#ifdef _WIN32 + LOGI(" - \"audio=DEVICE_NAME\" (for dshow)"); + LOGI(" - Try running 'ffmpeg -list_devices true -f dshow -i dummy' to see available devices"); +#else + LOGI(" - \"default\" (default ALSA device)"); + LOGI(" - \"hw:0,0\" (hardware device)"); + LOGI(" - Try running 'arecord -L' to list available devices"); +#endif + return; + } + + if (device_list->nb_devices == 0) { + LOGI(" No audio input devices found."); + } else { + for (int i = 0; i < device_list->nb_devices; i++) { + AVDeviceInfo *device = device_list->devices[i]; + if (device->device_description) { + LOGI(" %s (%s)", device->device_name, device->device_description); + } else { + LOGI(" %s", device->device_name); + } + } + } + + avdevice_free_list_devices(&device_list); + + LOGI("How to use:"); + LOGI(" Pass the device names shown above as --client-audio-source "); + LOGI("Common microphone devices:"); +#ifdef _WIN32 + LOGI(" - \"audio=DEVICE_NAME\" (use the exact name from the list)"); +#else + LOGI(" - \"default\" (usually your default microphone)"); + LOGI(" - \"hw:CARD,DEV\" devices are hardware devices"); + LOGI(" - \"pulse\" uses PulseAudio (if available)"); + LOGI(" Tip: Devices with \"capture\", \"input\", or \"microphone\" are likely input devices"); +#endif + LOGI("Examples:"); + LOGI(" scrcpy --client-audio-source default"); +#ifdef _WIN32 + LOGI(" scrcpy --client-audio-source \"audio=Microphone (Realtek Audio)\""); +#else + LOGI(" scrcpy --client-audio-source \"hw:0,0\""); +#endif + LOGI(" scrcpy --client-audio-source file:///path/to/audio.mp3"); +} + +int +sc_microphone_run(void *data) { + struct sc_microphone_params *params = (struct sc_microphone_params *)data; + sc_socket mic_socket = params->socket; + const char *audio_source = params->audio_source; + + int ret = 1; + const char *input_path = NULL; + bool is_file = parse_audio_source(audio_source, &input_path); + + AVInputFormat *input_format = NULL; + AVFormatContext *fmt_ctx = NULL; + AVCodecContext *in_codec_ctx = NULL; + AVCodecContext *opus_ctx = NULL; + SwrContext *swr_ctx = NULL; + AVPacket *in_pkt = NULL; + AVPacket *out_pkt = NULL; + AVFrame *in_frame = NULL; + AVFrame *out_frame = NULL; + AVAudioFifo *fifo = NULL; + + if (is_file) { + // Open audio file (MP3, OGG, WAV, etc.) + LOGD("Opening audio file: %s", input_path); + if (avformat_open_input(&fmt_ctx, input_path, NULL, NULL) < 0) { + LOGE("Could not open audio file '%s'", input_path); + goto cleanup; + } + } else { + // Open audio device + const char *format_name = detect_audio_format(); + LOGD("Using audio format: %s, device: %s", format_name, input_path); + + input_format = av_find_input_format(format_name); + if (!input_format) { + LOGE("Could not find audio input format '%s'", format_name); + goto cleanup; + } + + if (avformat_open_input(&fmt_ctx, input_path, input_format, NULL) < 0) { + LOGE("Could not open audio device '%s'", input_path); + goto cleanup; + } + } + + if (avformat_find_stream_info(fmt_ctx, NULL) < 0) { + LOGE("Could not read stream info"); + goto cleanup; + } + + int audio_stream_index = 0; + AVCodecParameters *in_codecpar = + fmt_ctx->streams[audio_stream_index]->codecpar; + AVCodec *in_codec = avcodec_find_decoder(in_codecpar->codec_id); + if (!in_codec) { + LOGE("Input codec not found"); + goto cleanup; + } + + in_codec_ctx = avcodec_alloc_context3(in_codec); + if (!in_codec_ctx) { + LOGE("Could not allocate input codec context"); + goto cleanup; + } + + avcodec_parameters_to_context(in_codec_ctx, in_codecpar); + if (avcodec_open2(in_codec_ctx, in_codec, NULL) < 0) { + LOGE("Could not open input codec"); + goto cleanup; + } + + LOGD("Input: sample_fmt=%d, sample_rate=%d, channels=%d", + in_codec_ctx->sample_fmt, in_codec_ctx->sample_rate, + in_codec_ctx->ch_layout.nb_channels); + + // Setup Opus encoder + AVCodec *opus_codec = avcodec_find_encoder(AV_CODEC_ID_OPUS); + if (!opus_codec) { + LOGE("Opus encoder not found"); + goto cleanup; + } + + opus_ctx = avcodec_alloc_context3(opus_codec); + if (!opus_ctx) { + LOGE("Could not allocate Opus codec context"); + goto cleanup; + } + + opus_ctx->sample_fmt = AV_SAMPLE_FMT_S16; // Opus uses 16-bit PCM + opus_ctx->sample_rate = 48000; // Opus standard sample rate + opus_ctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO; + opus_ctx->bit_rate = 128000; // 128 kbps + + if (avcodec_open2(opus_ctx, opus_codec, NULL) < 0) { + LOGE("Could not open Opus encoder"); + goto cleanup; + } + LOGD("opus_ctx->frame_size = %d", opus_ctx->frame_size); + + LOGD("Opus encoder: sample_rate=%d, channels=%d, bit_rate=%" PRId64 " fmt=%d", + opus_ctx->sample_rate, opus_ctx->ch_layout.nb_channels, + opus_ctx->bit_rate, opus_ctx->sample_fmt); + + // Setup resampler (ALSA format -> Opus format) + swr_ctx = swr_alloc(); + if (!swr_ctx) { + LOGE("Could not allocate resampler"); + goto cleanup; + } + + swr_alloc_set_opts2(&swr_ctx, &opus_ctx->ch_layout, opus_ctx->sample_fmt, + opus_ctx->sample_rate, &in_codec_ctx->ch_layout, + in_codec_ctx->sample_fmt, in_codec_ctx->sample_rate, 0, + NULL); + if (swr_init(swr_ctx) < 0) { + LOGE("Could not initialize resampler"); + goto cleanup; + } + + in_pkt = av_packet_alloc(); + out_pkt = av_packet_alloc(); + in_frame = av_frame_alloc(); + out_frame = av_frame_alloc(); + if (!in_pkt || !out_pkt || !in_frame || !out_frame) { + LOGE("Could not allocate packets/frames"); + goto cleanup; + } + + out_frame->format = opus_ctx->sample_fmt; + out_frame->ch_layout = opus_ctx->ch_layout; + out_frame->sample_rate = opus_ctx->sample_rate; + out_frame->nb_samples = opus_ctx->frame_size; + LOGD("setting nb_samples to %d", opus_ctx->frame_size); + if (av_frame_get_buffer(out_frame, 0) < 0) { + LOGE("Could not allocate frame buffer"); + goto cleanup; + } + + fifo = av_audio_fifo_alloc(opus_ctx->sample_fmt, opus_ctx->ch_layout.nb_channels, + opus_ctx->frame_size * 2); + if (!fifo) { + LOGE("Could not allocate audio FIFO"); + goto cleanup; + } + + LOGI("Recording audio with Opus encoding..."); + if (is_file) { + LOGI("File will loop continuously. Press Ctrl+C to stop."); + } + + // For file playback: track timing to simulate real-time playback + // Each Opus frame is opus_ctx->frame_size samples at opus_ctx->sample_rate + int64_t start_time = -1; + int64_t opus_frames_sent = 0; + // Calculate frame duration in microseconds: (frame_size / sample_rate) * 1000000 + int64_t opus_frame_duration_us = (opus_ctx->frame_size * 1000000LL) / opus_ctx->sample_rate; + + // Flag to track if we should stop (e.g., socket closed) + bool should_stop = false; + + // Main loop - for files, this will restart from beginning when reaching EOF + while (!should_stop) { + int read_ret = av_read_frame(fmt_ctx, in_pkt); + + // If EOF reached on a file, seek back to start and continue + if (read_ret == AVERROR_EOF && is_file) { + LOGD("File ended, looping back to start"); + av_seek_frame(fmt_ctx, audio_stream_index, 0, AVSEEK_FLAG_BACKWARD); + avcodec_flush_buffers(in_codec_ctx); + continue; + } + + if (read_ret < 0) { + // EAGAIN means no data available yet, retry + if (read_ret == AVERROR(EAGAIN)) { + av_usleep(1000); + continue; + } + if (!is_file) { + char errbuf[128]; + av_strerror(read_ret, errbuf, sizeof(errbuf)); + LOGD("av_read_frame error: %d (%s)", read_ret, errbuf); + break; + } + continue; + } + if (avcodec_send_packet(in_codec_ctx, in_pkt) < 0) + continue; + + while (avcodec_receive_frame(in_codec_ctx, in_frame) >= 0) { + // Resample to Opus format + int out_samples = + swr_convert(swr_ctx, (uint8_t **)out_frame->data, opus_ctx->frame_size, + (const uint8_t **)in_frame->data, in_frame->nb_samples); + + if (out_samples <= 0) + continue; + + av_audio_fifo_write(fifo, (void **)out_frame->data, out_samples); + + while (av_audio_fifo_size(fifo) >= opus_ctx->frame_size) { + av_audio_fifo_read(fifo, (void **)out_frame->data, + opus_ctx->frame_size); + + out_frame->nb_samples = opus_ctx->frame_size; + out_frame->pts = av_rescale_q(in_frame->pts, in_codec_ctx->time_base, + opus_ctx->time_base); + + // Encode to Opus + if (avcodec_send_frame(opus_ctx, out_frame) < 0) + continue; + + while (avcodec_receive_packet(opus_ctx, out_pkt) >= 0) { + uint32_t size = out_pkt->size; + uint8_t size_buf[4]; + sc_write32be(size_buf, size); + + // Send packet size (4 bytes, big-endian) + ssize_t sent = net_send_all(mic_socket, size_buf, 4); + if (sent < 4) { + LOGD("Failed to send packet size, socket closed"); + should_stop = true; + av_packet_unref(out_pkt); + break; + } + + // Send Opus packet + sent = net_send_all(mic_socket, out_pkt->data, out_pkt->size); + if (sent < (ssize_t)out_pkt->size) { + LOGD("Failed to send packet data, socket closed"); + should_stop = true; + av_packet_unref(out_pkt); + break; + } + + av_packet_unref(out_pkt); + + // For file input, add timing control AFTER sending + if (is_file) { + if (start_time < 0) { + start_time = av_gettime_relative(); + } + + opus_frames_sent++; + + // Calculate expected time based on Opus frames sent + int64_t expected_time_us = opus_frames_sent * opus_frame_duration_us; + int64_t elapsed_time_us = av_gettime_relative() - start_time; + int64_t sleep_time_us = expected_time_us - elapsed_time_us; + + if (sleep_time_us > 0) { + av_usleep(sleep_time_us); + } + } + } + } + } + av_packet_unref(in_pkt); + } + + // Only flush and cleanup for device input + // For file input, the loop is infinite (controlled by user/scrcpy termination) + if (!is_file) { + // Flush the decoder + avcodec_send_packet(in_codec_ctx, NULL); + while (avcodec_receive_frame(in_codec_ctx, in_frame) >= 0) { + int out_samples = + swr_convert(swr_ctx, (uint8_t **)out_frame->data, opus_ctx->frame_size, + (const uint8_t **)in_frame->data, in_frame->nb_samples); + if (out_samples > 0) { + av_audio_fifo_write(fifo, (void **)out_frame->data, out_samples); + } + } + + // Flush any remaining audio in the FIFO + while (av_audio_fifo_size(fifo) > 0) { + int samples_to_read = av_audio_fifo_size(fifo); + if (samples_to_read > opus_ctx->frame_size) { + samples_to_read = opus_ctx->frame_size; + } + + av_audio_fifo_read(fifo, (void **)out_frame->data, samples_to_read); + out_frame->nb_samples = samples_to_read; + + if (avcodec_send_frame(opus_ctx, out_frame) >= 0) { + while (avcodec_receive_packet(opus_ctx, out_pkt) >= 0) { + uint32_t size = out_pkt->size; + uint8_t size_buf[4]; + sc_write32be(size_buf, size); + + ssize_t sent = net_send_all(mic_socket, size_buf, 4); + if (sent < 4) { + av_packet_unref(out_pkt); + break; // Socket closed, skip remaining flush + } + + sent = net_send_all(mic_socket, out_pkt->data, out_pkt->size); + if (sent < (ssize_t)out_pkt->size) { + av_packet_unref(out_pkt); + break; // Socket closed, skip remaining flush + } + + av_packet_unref(out_pkt); + } + } + } + + // Flush the Opus encoder + avcodec_send_frame(opus_ctx, NULL); + while (avcodec_receive_packet(opus_ctx, out_pkt) >= 0) { + uint32_t size = out_pkt->size; + uint8_t size_buf[4]; + sc_write32be(size_buf, size); + + ssize_t sent = net_send_all(mic_socket, size_buf, 4); + if (sent < 4) { + av_packet_unref(out_pkt); + break; // Socket closed, skip remaining flush + } + + sent = net_send_all(mic_socket, out_pkt->data, out_pkt->size); + if (sent < (ssize_t)out_pkt->size) { + av_packet_unref(out_pkt); + break; // Socket closed, skip remaining flush + } + + av_packet_unref(out_pkt); + } + + // Give the server time to process remaining packets before closing + av_usleep(200000); // 200ms + } + + LOGD("Audio streaming ended"); + ret = 0; // Success + +cleanup: + // Cleanup all resources in reverse order of allocation + if (fifo) { + av_audio_fifo_free(fifo); + } + if (swr_ctx) { + swr_free(&swr_ctx); + } + if (out_frame) { + av_frame_free(&out_frame); + } + if (in_frame) { + av_frame_free(&in_frame); + } + if (out_pkt) { + av_packet_free(&out_pkt); + } + if (in_pkt) { + av_packet_free(&in_pkt); + } + if (opus_ctx) { + avcodec_free_context(&opus_ctx); + } + if (in_codec_ctx) { + avcodec_free_context(&in_codec_ctx); + } + if (fmt_ctx) { + avformat_close_input(&fmt_ctx); + } + + // Close the socket + if (mic_socket != SC_SOCKET_NONE) { + if (!net_close(mic_socket)) { + LOGW("Could not close microphone socket"); + } else { + LOGD("Microphone socket closed"); + } + } + + return ret; +} diff --git a/app/src/client_audio.h b/app/src/client_audio.h new file mode 100644 index 0000000000..021071b337 --- /dev/null +++ b/app/src/client_audio.h @@ -0,0 +1,17 @@ +#ifndef SC_CLIENT_AUDIO_H +#define SC_CLIENT_AUDIO_H + +#include "util/net.h" + +struct sc_microphone_params { + sc_socket socket; + const char *audio_source; +}; + +void +sc_microphone_list_audio_sources(void); + +int +sc_microphone_run(void *data); + +#endif diff --git a/app/src/main.c b/app/src/main.c index 3ab03243f7..2bedde6243 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -2,7 +2,7 @@ #include #include -#ifdef HAVE_V4L2 +#ifdef HAVE_CLIENT_AUDIO # include #endif #include @@ -17,6 +17,9 @@ #include "util/net.h" #include "util/thread.h" #include "version.h" +#ifdef HAVE_CLIENT_AUDIO +# include "client_audio.h" +#endif #ifdef _WIN32 #include @@ -40,6 +43,7 @@ main_scrcpy(int argc, char *argv[]) { .help = false, .version = false, .pause_on_exit = SC_PAUSE_ON_EXIT_UNDEFINED, + .list_audio_sources = false, }; #ifndef NDEBUG @@ -67,19 +71,24 @@ main_scrcpy(int argc, char *argv[]) { goto end; } - // The current thread is the main thread - SC_MAIN_THREAD_ID = sc_thread_get_id(); - #ifdef SCRCPY_LAVF_REQUIRES_REGISTER_ALL av_register_all(); #endif -#ifdef HAVE_V4L2 - if (args.opts.v4l2_device) { - avdevice_register_all(); +#ifdef HAVE_CLIENT_AUDIO + //needed for capturing microphone and listing audio sources + avdevice_register_all(); + + if (args.list_audio_sources) { + sc_microphone_list_audio_sources(); + ret = SCRCPY_EXIT_SUCCESS; + goto end; } #endif + // The current thread is the main thread + SC_MAIN_THREAD_ID = sc_thread_get_id(); + if (!net_init()) { ret = SCRCPY_EXIT_FAILURE; goto end; diff --git a/app/src/options.c b/app/src/options.c index 95d023888b..710ecc08cc 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -103,6 +103,7 @@ const struct scrcpy_options scrcpy_options_default = { .power_on = true, .video = true, .audio = true, + .client_audio_source = NULL, .require_audio = false, .kill_adb_on_close = false, .camera_high_speed = false, diff --git a/app/src/options.h b/app/src/options.h index 9a37ec6fa9..f8c1000ada 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -313,6 +313,7 @@ struct scrcpy_options { bool power_on; bool video; bool audio; + const char *client_audio_source; bool require_audio; bool kill_adb_on_close; bool camera_high_speed; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 0e63d2ab91..0fed2d539d 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -26,6 +26,9 @@ #include "recorder.h" #include "screen.h" #include "server.h" +#ifdef HAVE_CLIENT_AUDIO +# include "client_audio.h" +#endif #include "uhid/gamepad_uhid.h" #include "uhid/keyboard_uhid.h" #include "uhid/mouse_uhid.h" @@ -61,6 +64,10 @@ struct scrcpy { #endif struct sc_controller controller; struct sc_file_pusher file_pusher; +#ifdef HAVE_CLIENT_AUDIO + struct sc_microphone_params microphone_params; + sc_thread microphone_thread; +#endif #ifdef HAVE_USB struct sc_usb usb; struct sc_aoa aoa; @@ -441,6 +448,11 @@ scrcpy(struct scrcpy_options *options) { .display_ime_policy = options->display_ime_policy, .video = options->video, .audio = options->audio, +#ifdef HAVE_CLIENT_AUDIO + .client_audio = options->client_audio_source != NULL, +#else + .client_audio = false, +#endif .audio_dup = options->audio_dup, .show_touches = options->show_touches, .stay_awake = options->stay_awake, @@ -882,6 +894,20 @@ scrcpy(struct scrcpy_options *options) { audio_demuxer_started = true; } +#ifdef HAVE_CLIENT_AUDIO + bool microphone_started = false; + if (options->client_audio_source) { + s->microphone_params.socket = s->server.client_mic_socket; + s->microphone_params.audio_source = options->client_audio_source; + + bool ok = sc_thread_create(&s->microphone_thread, sc_microphone_run, "scrcpy-clnt-mic", &s->microphone_params); + if (!ok) { + goto end; + } + microphone_started = true; + } +#endif + // If the device screen is to be turned off, send the control message after // everything is set up if (options->control && options->turn_screen_off) { @@ -1014,6 +1040,12 @@ scrcpy(struct scrcpy_options *options) { sc_demuxer_join(&s->audio_demuxer); } +#ifdef HAVE_CLIENT_AUDIO + if (microphone_started) { + sc_thread_join(&s->microphone_thread, NULL); + } +#endif + #ifdef HAVE_V4L2 if (v4l2_sink_initialized) { sc_v4l2_sink_destroy(&s->v4l2_sink); diff --git a/app/src/server.c b/app/src/server.c index e108ead0ed..f67d16ac93 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -11,6 +11,7 @@ #include "util/env.h" #include "util/file.h" #include "util/log.h" +#include "util/net.h" #include "util/net_intr.h" #include "util/process.h" #include "util/str.h" @@ -271,6 +272,9 @@ execute_server(struct sc_server *server, if (params->video_bit_rate) { ADD_PARAM("video_bit_rate=%" PRIu32, params->video_bit_rate); } + if (params->client_audio) { + ADD_PARAM("client_audio=true"); + } if (!params->audio) { ADD_PARAM("audio=false"); } @@ -564,6 +568,7 @@ sc_server_init(struct sc_server *server, const struct sc_server_params *params, server->video_socket = SC_SOCKET_NONE; server->audio_socket = SC_SOCKET_NONE; server->control_socket = SC_SOCKET_NONE; + server->client_mic_socket = SC_SOCKET_NONE; sc_adb_tunnel_init(&server->tunnel); @@ -606,10 +611,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { bool video = server->params.video; bool audio = server->params.audio; bool control = server->params.control; + bool client_audio = server->params.client_audio; sc_socket video_socket = SC_SOCKET_NONE; sc_socket audio_socket = SC_SOCKET_NONE; sc_socket control_socket = SC_SOCKET_NONE; + sc_socket client_audio_socket = SC_SOCKET_NONE; if (!tunnel->forward) { if (video) { video_socket = @@ -634,6 +641,14 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { goto fail; } } + + if (client_audio) { + client_audio_socket = + net_accept_intr(&server->intr, tunnel->server_socket); + if (client_audio_socket == SC_SOCKET_NONE) { + goto fail; + } + } } else { uint32_t tunnel_host = server->params.tunnel_host; if (!tunnel_host) { @@ -688,23 +703,44 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { } } } + + if (client_audio) { + if (!video && !audio && !control) { + client_audio_socket = first_socket; + } else { + client_audio_socket = net_socket(); + if (client_audio_socket == SC_SOCKET_NONE) { + goto fail; + } + bool ok = net_connect_intr(&server->intr, client_audio_socket, tunnel_host, tunnel_port); + if (!ok) { + goto fail; + } + } + } } + // Disable Nagle's algorithm for the control socket + // (it only impacts the sending side, so it is useless to set it + // for the other sockets) if (control_socket != SC_SOCKET_NONE) { - // Disable Nagle's algorithm for the control socket - // (it only impacts the sending side, so it is useless to set it - // for the other sockets) bool ok = net_set_tcp_nodelay(control_socket, true); (void) ok; // error already logged } + if (client_audio_socket != SC_SOCKET_NONE) { + bool ok = net_set_tcp_nodelay(client_audio_socket, true); + (void) ok; // error already logged + } + // we don't need the adb tunnel anymore sc_adb_tunnel_close(tunnel, &server->intr, serial, server->device_socket_name); sc_socket first_socket = video ? video_socket : audio ? audio_socket - : control_socket; + : control ? control_socket + : client_audio_socket; // The sockets will be closed on stop if device_read_info() fails bool ok = device_read_info(&server->intr, first_socket, info); @@ -715,10 +751,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { assert(!video || video_socket != SC_SOCKET_NONE); assert(!audio || audio_socket != SC_SOCKET_NONE); assert(!control || control_socket != SC_SOCKET_NONE); + assert(!client_audio || client_audio_socket != SC_SOCKET_NONE); server->video_socket = video_socket; server->audio_socket = audio_socket; server->control_socket = control_socket; + server->client_mic_socket = client_audio_socket; return true; @@ -741,6 +779,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { } } + if (client_audio_socket != SC_SOCKET_NONE) { + if (!net_close(client_audio_socket)) { + LOGW("Could not close microphone socket"); + } + } + if (tunnel->enabled) { // Always leave this function with tunnel disabled sc_adb_tunnel_close(tunnel, &server->intr, serial, @@ -1130,6 +1174,11 @@ run_server(void *data) { net_interrupt(server->control_socket); } + if (server->client_mic_socket != SC_SOCKET_NONE) { + // There is no control_socket if --no-microphone is set + net_interrupt(server->client_mic_socket); + } + // Give some delay for the server to terminate properly #define WATCHDOG_DELAY SC_TICK_FROM_SEC(1) sc_tick deadline = sc_tick_now() + WATCHDOG_DELAY; @@ -1198,6 +1247,9 @@ sc_server_destroy(struct sc_server *server) { if (server->control_socket != SC_SOCKET_NONE) { net_close(server->control_socket); } + if (server->client_mic_socket != SC_SOCKET_NONE) { + net_close(server->client_mic_socket); + } free(server->serial); free(server->device_socket_name); diff --git a/app/src/server.h b/app/src/server.h index 252bb2bcb0..91ab8f88e7 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -55,6 +55,7 @@ struct sc_server_params { enum sc_display_ime_policy display_ime_policy; bool video; bool audio; + bool client_audio; bool audio_dup; bool show_touches; bool stay_awake; @@ -94,6 +95,7 @@ struct sc_server { sc_socket video_socket; sc_socket audio_socket; + sc_socket client_mic_socket; sc_socket control_socket; const struct sc_server_callbacks *cbs; diff --git a/app/tests/test_cli.c b/app/tests/test_cli.c index de605cb964..886f91f955 100644 --- a/app/tests/test_cli.c +++ b/app/tests/test_cli.c @@ -11,6 +11,7 @@ static void test_flag_version(void) { .opts = scrcpy_options_default, .help = false, .version = false, + .list_audio_sources = false, }; char *argv[] = {"scrcpy", "-v"}; @@ -26,6 +27,7 @@ static void test_flag_help(void) { .opts = scrcpy_options_default, .help = false, .version = false, + .list_audio_sources = false, }; char *argv[] = {"scrcpy", "-v"}; @@ -41,6 +43,7 @@ static void test_options(void) { .opts = scrcpy_options_default, .help = false, .version = false, + .list_audio_sources = false, }; char *argv[] = { @@ -101,6 +104,7 @@ static void test_options2(void) { .opts = scrcpy_options_default, .help = false, .version = false, + .list_audio_sources = false, }; char *argv[] = { diff --git a/meson_options.txt b/meson_options.txt index fd347734dc..c5ce52d060 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -6,3 +6,4 @@ option('static', type: 'boolean', value: false, description: 'Use static depende option('server_debugger', type: 'boolean', value: false, description: 'Run a server debugger and wait for a client to be attached') option('v4l2', type: 'boolean', value: true, description: 'Enable V4L2 feature when supported') option('usb', type: 'boolean', value: true, description: 'Enable HID/OTG features when supported') +option('client_audio', type: 'boolean', value: true, description: 'Enable client audio forwarding (requires libavdevice)') diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioInjector.java b/server/src/main/java/com/genymobile/scrcpy/AudioInjector.java new file mode 100644 index 0000000000..5f429b7e82 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioInjector.java @@ -0,0 +1,166 @@ +package com.genymobile.scrcpy; + +import android.annotation.SuppressLint; +import android.content.Context; +import android.media.AudioAttributes; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioTrack; +import android.media.MediaRecorder; + +import com.genymobile.scrcpy.util.Ln; + +import java.io.PipedInputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.util.Objects; + +/** + * Injects audio from the client computer into the Android device's microphone + * using AudioPolicy APIs via reflection. + * + * Based on: https://github.com/Genymobile/scrcpy/issues/3880#issuecomment-1595722119 + */ +public final class AudioInjector { + + private AudioInjector() { + } + + private static AudioAttributes createAudioAttributes(int capturePreset) throws Exception { + AudioAttributes.Builder audioAttributesBuilder = new AudioAttributes.Builder(); + Method setCapturePresetMethod = + audioAttributesBuilder.getClass().getDeclaredMethod("setCapturePreset", int.class); + setCapturePresetMethod.invoke(audioAttributesBuilder, capturePreset); + return audioAttributesBuilder.build(); + } + + /** + * Injects audio from a PipedInputStream into the device's microphone. + * + * @param pis The PipedInputStream containing PCM audio data to inject + * @throws Exception if audio injection setup fails + */ + public static void injectAudio(PipedInputStream pis) throws Exception { + Context systemContext = Workarounds.getSystemContext(); + Objects.requireNonNull(systemContext); + + // var audioMixRuleBuilder = new AudioMixingRule.Builder(); + @SuppressLint("PrivateApi") + Class audioMixRuleBuilderClass = + Class.forName("android.media.audiopolicy.AudioMixingRule$Builder"); + Object audioMixRuleBuilder = audioMixRuleBuilderClass.getDeclaredConstructor().newInstance(); + + try { + // Added in Android 13, but previous versions don't work because lack of permission. + // audioMixRuleBuilder.setTargetMixRole(MIX_ROLE_INJECTOR); + Method setTargetMixRoleMethod = + audioMixRuleBuilder.getClass().getDeclaredMethod("setTargetMixRole", int.class); + int MIX_ROLE_INJECTOR = 1; + setTargetMixRoleMethod.invoke(audioMixRuleBuilder, MIX_ROLE_INJECTOR); + } catch (Exception ignored) { + } + + Method addMixRuleMethod = audioMixRuleBuilder.getClass() + .getDeclaredMethod("addMixRule", int.class, Object.class); + int RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET = 0x1 << 1; + + // Add mix rules for various capture presets to intercept all microphone capture + addMixRuleMethod.invoke(audioMixRuleBuilder, RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET, + createAudioAttributes(MediaRecorder.AudioSource.DEFAULT)); + addMixRuleMethod.invoke(audioMixRuleBuilder, RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET, + createAudioAttributes(MediaRecorder.AudioSource.MIC)); + addMixRuleMethod.invoke(audioMixRuleBuilder, RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET, + createAudioAttributes( + MediaRecorder.AudioSource.VOICE_COMMUNICATION)); + addMixRuleMethod.invoke(audioMixRuleBuilder, RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET, + createAudioAttributes(MediaRecorder.AudioSource.UNPROCESSED)); + + // var audioMixingRule = audioMixRuleBuilder.build(); + Method audioMixRuleBuildMethod = audioMixRuleBuilder.getClass().getDeclaredMethod("build"); + Object audioMixingRule = audioMixRuleBuildMethod.invoke(audioMixRuleBuilder); + Objects.requireNonNull(audioMixingRule); + + // var audioMixBuilder = new AudioMix.Builder(audioMixingRule); + @SuppressLint("PrivateApi") + Class audioMixBuilderClass = Class.forName("android.media.audiopolicy.AudioMix$Builder"); + Constructor audioMixBuilderConstructor = + audioMixBuilderClass.getDeclaredConstructor(audioMixingRule.getClass()); + Object audioMixBuilder = audioMixBuilderConstructor.newInstance(audioMixingRule); + + Object audioFormat = new AudioFormat.Builder().setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(48000) + .setChannelMask(AudioFormat.CHANNEL_IN_STEREO) + .build(); + + // audioMixBuilder.setFormat(audioFormat); + Method setFormatMethod = + audioMixBuilder.getClass().getDeclaredMethod("setFormat", AudioFormat.class); + setFormatMethod.invoke(audioMixBuilder, audioFormat); + + // audioMixBuilder.setRouteFlags(ROUTE_FLAG_LOOP_BACK); + Method setRouteFlagsMethod = + audioMixBuilder.getClass().getDeclaredMethod("setRouteFlags", int.class); + int ROUTE_FLAG_LOOP_BACK = 0x1 << 1; + setRouteFlagsMethod.invoke(audioMixBuilder, ROUTE_FLAG_LOOP_BACK); + + // var audioMix = audioMixBuilder.build(); + Method audioMixBuildMethod = audioMixBuilder.getClass().getDeclaredMethod("build"); + Object audioMix = audioMixBuildMethod.invoke(audioMixBuilder); + Objects.requireNonNull(audioMix); + + // var audioPolicyBuilder = new AudioPolicy.Builder(systemContext); + @SuppressLint("PrivateApi") + Class audioPolicyBuilderClass = + Class.forName("android.media.audiopolicy.AudioPolicy$Builder"); + Constructor audioPolicyBuilderConstructor = + audioPolicyBuilderClass.getDeclaredConstructor(Context.class); + Object audioPolicyBuilder = audioPolicyBuilderConstructor.newInstance(systemContext); + + // audioPolicyBuilder.addMix(audioMix); + Method addMixMethod = + audioPolicyBuilder.getClass().getDeclaredMethod("addMix", audioMix.getClass()); + addMixMethod.invoke(audioPolicyBuilder, audioMix); + + // var audioPolicy = audioPolicyBuilder.build(); + Method audioPolicyBuildMethod = audioPolicyBuilder.getClass().getDeclaredMethod("build"); + Object audioPolicy = audioPolicyBuildMethod.invoke(audioPolicyBuilder); + Objects.requireNonNull(audioPolicy); + + Object audioManager = (AudioManager) systemContext.getSystemService(AudioManager.class); + + // audioManager.registerAudioPolicy(audioPolicy); + Method registerAudioPolicyMethod = audioManager.getClass() + .getDeclaredMethod("registerAudioPolicy", audioPolicy.getClass()); + // noinspection DataFlowIssue + int result = (int) registerAudioPolicyMethod.invoke(audioManager, audioPolicy); + + if (result != 0) { + Ln.d("registerAudioPolicy failed"); + return; + } + + // var audioTrack = audioPolicy.createAudioTrackSource(audioMix); + Method createAudioTrackSourceMethod = audioPolicy.getClass() + .getDeclaredMethod("createAudioTrackSource", audioMix.getClass()); + AudioTrack audioTrack = (AudioTrack) createAudioTrackSourceMethod.invoke(audioPolicy, audioMix); + Objects.requireNonNull(audioTrack); + + audioTrack.play(); + + new Thread(() -> { + byte[] audioBuffer = new byte[4096]; + while (true) { + try { + int bytesRead = pis.read(audioBuffer); + if (bytesRead <= 0) { + break; + } + audioTrack.write(audioBuffer, 0, bytesRead); + } catch (Exception e) { + Ln.e("Audio injection error", e); + break; + } + } + }, "client-audio-injector").start(); + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 450d4cf743..63e7c8561e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -26,6 +26,7 @@ public class Options { private int scid = -1; // 31-bit non-negative value, or -1 private boolean video = true; private boolean audio = true; + private boolean client_audio = false; private int maxSize; private int minSizeAlignment = 1; private VideoCodec videoCodec = VideoCodec.H264; @@ -100,6 +101,10 @@ public boolean getAudio() { return audio; } + public boolean getClientAudio() { + return client_audio; + } + public int getMaxSize() { return maxSize; } @@ -343,6 +348,9 @@ public static Options parse(String... args) { case "audio": options.audio = Boolean.parseBoolean(value); break; + case "client_audio": + options.client_audio = Boolean.parseBoolean(value); + break; case "video_codec": VideoCodec videoCodec = VideoCodec.findByName(value); if (videoCodec == null) { @@ -555,6 +563,12 @@ public static Options parse(String... args) { options.displayId = Device.DISPLAY_ID_NONE; } + if (options.client_audio && + (options.audioSource == AudioSource.VOICE_CALL || + options.audioSource == AudioSource.VOICE_CALL_UPLINK)) { + throw new IllegalArgumentException("client_audio is incompatible with audio_source=voice-call and audio_source=voice-call-uplink"); + } + return options; } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index e3f1680de8..4b223e9b13 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -1,7 +1,10 @@ package com.genymobile.scrcpy; +import android.net.LocalSocket; + import com.genymobile.scrcpy.audio.AudioCapture; import com.genymobile.scrcpy.audio.AudioCodec; +import com.genymobile.scrcpy.audio.AudioDecoder; import com.genymobile.scrcpy.audio.AudioDirectCapture; import com.genymobile.scrcpy.audio.AudioEncoder; import com.genymobile.scrcpy.audio.AudioPlaybackCapture; @@ -29,8 +32,12 @@ import android.os.Looper; import android.system.Os; +import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; @@ -96,13 +103,14 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc boolean control = options.getControl(); boolean video = options.getVideo(); boolean audio = options.getAudio(); + boolean client_audio = options.getClientAudio(); boolean sendDummyByte = options.getSendDummyByte(); Workarounds.apply(); List asyncProcessors = new ArrayList<>(); - DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, video, audio, control, sendDummyByte); + DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, video, audio, control, client_audio, sendDummyByte); try { if (options.getSendDeviceMeta()) { connection.sendDeviceMeta(Device.getDeviceName()); @@ -159,6 +167,21 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } } + if (client_audio) { + try { + LocalSocket s = connection.getClientAudioSocket(); + InputStream is = s.getInputStream(); + BufferedInputStream bis = new BufferedInputStream(is); + PipedOutputStream pos = new PipedOutputStream(); + PipedInputStream pis = new PipedInputStream(pos, 500 * 1024); + AudioDecoder decoder = new AudioDecoder(); + decoder.start(bis, pos); + AudioInjector.injectAudio(pis); + } catch (Exception e) { + Ln.e("Client audio injection error", e); + } + } + Completion completion = new Completion(asyncProcessors.size()); for (AsyncProcessor asyncProcessor : asyncProcessors) { asyncProcessor.start((fatalError) -> { diff --git a/server/src/main/java/com/genymobile/scrcpy/audio/AudioDecoder.java b/server/src/main/java/com/genymobile/scrcpy/audio/AudioDecoder.java new file mode 100644 index 0000000000..23af948444 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/audio/AudioDecoder.java @@ -0,0 +1,134 @@ +package com.genymobile.scrcpy.audio; + +import android.media.MediaCodec; +import android.media.MediaFormat; +import com.genymobile.scrcpy.util.Ln; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class AudioDecoder { + private MediaCodec decoder; + private boolean running = false; + + public void start(BufferedInputStream bis, OutputStream pcmOutput) throws IOException { + // Initialize Opus decoder + decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_AUDIO_OPUS); + MediaFormat format = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_OPUS, + AudioConfig.SAMPLE_RATE, + AudioConfig.CHANNELS + ); + + // OpusHead structure for stereo 48kHz Opus + // Format: "OpusHead" + version(1) + channel_count(2) + pre_skip(312) + sample_rate(48000) + output_gain(0) + channel_mapping(0) + byte[] opusHead = new byte[] { + 0x4F, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64, // "OpusHead" + 0x01, // Version + 0x02, // Channel count (2 = stereo) + 0x38, 0x01, // Pre-skip (312 samples, little-endian) + (byte)0x80, (byte)0xBB, 0x00, 0x00, // Input sample rate (48000 Hz, little-endian) + 0x00, 0x00, // Output gain (0 dB) + 0x00 // Channel mapping family (0 = mono/stereo) + }; + format.setByteBuffer("csd-0", ByteBuffer.wrap(opusHead)); + + // CSD-1: Pre-skip in nanoseconds (312 samples * 1,000,000,000 / 48,000 = 6,500,000 ns) + long preSkipNs = 6500000L; + ByteBuffer csd1 = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + csd1.putLong(preSkipNs); + csd1.flip(); + format.setByteBuffer("csd-1", csd1); + + // CSD-2: Seek pre-roll in nanoseconds (80ms for Opus = 3840 samples at 48kHz) + long seekPreRollNs = 80000000L; + ByteBuffer csd2 = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + csd2.putLong(seekPreRollNs); + csd2.flip(); + format.setByteBuffer("csd-2", csd2); + + decoder.configure(format, null, null, 0); + decoder.start(); + + running = true; + + // Decoding loop + new Thread(() -> { + try { + decode(bis, pcmOutput); + } catch (Exception e) { + Ln.e("Opus decoder error", e); + } finally { + stop(); + } + }, "opus-decoder").start(); + } + + private void decode(BufferedInputStream bis, OutputStream pcmOutput) throws IOException { + byte[] sizeBuffer = new byte[4]; + MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + long presentationTime = 0; + + while (running) { + // Feed input packets (non-blocking) + int inputIndex = decoder.dequeueInputBuffer(0); + if (inputIndex >= 0) { + int bytesRead = bis.read(sizeBuffer); + if (bytesRead == 4) { + int packetSize = ((sizeBuffer[0] & 0xFF) << 24) | + ((sizeBuffer[1] & 0xFF) << 16) | + ((sizeBuffer[2] & 0xFF) << 8) | + (sizeBuffer[3] & 0xFF); + + if (packetSize > 0 && packetSize <= 100000) { + byte[] opusData = new byte[packetSize]; + int actualRead = bis.read(opusData); + if (actualRead == packetSize) { + ByteBuffer inputBuffer = decoder.getInputBuffer(inputIndex); + inputBuffer.clear(); + inputBuffer.put(opusData); + decoder.queueInputBuffer(inputIndex, 0, packetSize, presentationTime, 0); + // Increment presentation time (20ms per frame at 48kHz = 960 samples) + presentationTime += 20000; // microseconds + } + } + } else if (bytesRead == -1) { + // End of stream + decoder.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + break; + } + } + + // Drain available output + int outputIndex = decoder.dequeueOutputBuffer(bufferInfo, 0); + if (outputIndex >= 0) { + ByteBuffer outputBuffer = decoder.getOutputBuffer(outputIndex); + if (bufferInfo.size > 0) { + byte[] pcmData = new byte[bufferInfo.size]; + outputBuffer.position(bufferInfo.offset); + outputBuffer.limit(bufferInfo.offset + bufferInfo.size); + outputBuffer.get(pcmData); + pcmOutput.write(pcmData); + pcmOutput.flush(); + } + decoder.releaseOutputBuffer(outputIndex, false); + } else if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + // No output available yet, continue feeding input + } + } + } + + public void stop() { + running = false; + if (decoder != null) { + try { + decoder.stop(); + decoder.release(); + } catch (Exception e) {} + decoder = null; + } + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/device/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/device/DesktopConnection.java index db75aec671..9dea923035 100644 --- a/server/src/main/java/com/genymobile/scrcpy/device/DesktopConnection.java +++ b/server/src/main/java/com/genymobile/scrcpy/device/DesktopConnection.java @@ -12,6 +12,7 @@ import java.io.FileDescriptor; import java.io.IOException; import java.nio.charset.StandardCharsets; +import com.genymobile.scrcpy.util.Ln; public final class DesktopConnection implements Closeable { @@ -28,14 +29,19 @@ public final class DesktopConnection implements Closeable { private final LocalSocket controlSocket; private final ControlChannel controlChannel; - private DesktopConnection(LocalSocket videoSocket, LocalSocket audioSocket, LocalSocket controlSocket) throws IOException { + private final LocalSocket clientAudioSocket; + //private final FileDescriptor micFd; + + private DesktopConnection(LocalSocket videoSocket, LocalSocket audioSocket, LocalSocket controlSocket, LocalSocket clientAudioSocket) throws IOException { this.videoSocket = videoSocket; this.audioSocket = audioSocket; this.controlSocket = controlSocket; + this.clientAudioSocket = clientAudioSocket; videoFd = videoSocket != null ? videoSocket.getFileDescriptor() : null; audioFd = audioSocket != null ? audioSocket.getFileDescriptor() : null; controlChannel = controlSocket != null ? new ControlChannel(controlSocket) : null; + //micFd = clientAudioSocket != null ? clientAudioSocket.getFileDescriptor() : null; } private static LocalSocket connect(String abstractName) throws IOException { @@ -53,13 +59,14 @@ private static String getSocketName(int scid) { return SOCKET_NAME_PREFIX + String.format("_%08x", scid); } - public static DesktopConnection open(int scid, boolean tunnelForward, boolean video, boolean audio, boolean control, boolean sendDummyByte) + public static DesktopConnection open(int scid, boolean tunnelForward, boolean video, boolean audio, boolean control, boolean client_audio, boolean sendDummyByte) throws IOException { String socketName = getSocketName(scid); LocalSocket videoSocket = null; LocalSocket audioSocket = null; LocalSocket controlSocket = null; + LocalSocket clientAudioSocket = null; try { if (tunnelForward) { try (LocalServerSocket localServerSocket = new LocalServerSocket(socketName)) { @@ -87,6 +94,14 @@ public static DesktopConnection open(int scid, boolean tunnelForward, boolean vi sendDummyByte = false; } } + if (client_audio) { + clientAudioSocket = localServerSocket.accept(); + if (sendDummyByte) { + // send one byte so the client may read() to detect a connection error + clientAudioSocket.getOutputStream().write(0); + sendDummyByte = false; + } + } } } else { if (video) { @@ -98,6 +113,9 @@ public static DesktopConnection open(int scid, boolean tunnelForward, boolean vi if (control) { controlSocket = connect(socketName); } + if (client_audio) { + clientAudioSocket = connect(socketName); + } } } catch (IOException | RuntimeException e) { if (videoSocket != null) { @@ -109,10 +127,13 @@ public static DesktopConnection open(int scid, boolean tunnelForward, boolean vi if (controlSocket != null) { controlSocket.close(); } + if (clientAudioSocket != null) { + clientAudioSocket.close(); + } throw e; } - return new DesktopConnection(videoSocket, audioSocket, controlSocket); + return new DesktopConnection(videoSocket, audioSocket, controlSocket, clientAudioSocket); } private LocalSocket getFirstSocket() { @@ -122,7 +143,10 @@ private LocalSocket getFirstSocket() { if (audioSocket != null) { return audioSocket; } - return controlSocket; + if (controlSocket != null) { + return controlSocket; + } + return clientAudioSocket; } public void shutdown() throws IOException { @@ -172,6 +196,11 @@ public FileDescriptor getAudioFd() { return audioFd; } + public LocalSocket getClientAudioSocket() + { + return clientAudioSocket; + } + public ControlChannel getControlChannel() { return controlChannel; }