找手工活做注册网站,wordpress mdui,杭州 网站开发,wordpress做音乐网插件吗技术背景
随着物联网等行业的崛起#xff0c;越来越多的传统行业如虚拟仿真、航天工业、工业仿真、城市规划等#xff0c;对Linux下的生态构建#xff0c;有了更大的期望#xff0c;Linux平台下#xff0c;可选的直播推拉流解决方案相对Windows和移动端#xff0c;非常少…技术背景
随着物联网等行业的崛起越来越多的传统行业如虚拟仿真、航天工业、工业仿真、城市规划等对Linux下的生态构建有了更大的期望Linux平台下可选的直播推拉流解决方案相对Windows和移动端非常少基于Unity的Linux推送方案更是几无参考。本文以Unity3d环境下Linux平台推送Unity窗体和Unity采集的音频然后编码推送到RTMP服务器为例大概说下实现过程。
技术实现
本文以采集Unity窗体数据为例如果需要对接摄像头和屏幕亦可。简单来说高效率的获取到原始的窗体Texture拿到对应的RGB数据传给现有的RTMP推送模块音频的话只要获取到AudioClip数据实时读取然后传PCM数据。如需2路PCM数据混音可以采集两路AudioClip数据分别调用原生的Linux推送RTMP模块的PCM数据投递接口编码打包推送即可无图无真相
下图以Linux平台下unity3d环境采集窗体为方便测试加了个实时时间显示和unity声音为例整体延迟可达毫秒级。
音频的话可以循环采集播放、也可以播放完毕后加实时静音。 AudioClip数据采集如需audiosource数据播放完毕后填充静音帧可调用FillPcmMuteData()接口。 /// summary/// 获取AudioClip数据/// /summaryprivate void PostUnityAudioClipData(){//重新初始化TimerManager.UnRegister(this);audio_clip_info_ new AudioClipInfo();audio_clip_info_.audio_source_ GameObject.Find(Canvas/Panel/AudioSource).GetComponentAudioSource();if (audio_clip_info_.audio_source_ null){Debug.LogError(audio source is null..);return;}if (audio_clip_info_.audio_source_.clip null){Debug.LogError(audio clip is null..);return;}audio_clip_info_.audio_clip_ audio_clip_info_.audio_source_.clip;audio_clip_info_.audio_clip_offset_ 0;audio_clip_info_.audio_clip_length_ audio_clip_info_.audio_clip_.samples * audio_clip_info_.audio_clip_.channels;pcm_mute_data_ FillPcmMuteData(audio_clip_info_.audio_clip_);TimerManager.Register(this, 0.019999f, (PostPCMData), false);}
采集后的数据定时调用原生接口发送到原生接口
var sample_length sizeof(float) * pcm_sample.Length;pcm_data.data_ Marshal.AllocHGlobal(sample_length);
Marshal.Copy(pcm_sample, 0, pcm_data.data_, pcm_sample.Length);
pcm_data.size_ (uint)sample_length;publisher_wrapper_.OnPostAudioPCMFloatData(pcm_data.data_,pcm_data.size_,pcm_time_stamp_,pcm_data.sample_rate_,pcm_data.channels_,pcm_data.per_channel_sample_number_);Marshal.FreeHGlobal(pcm_data.data_);
pcm_data.data_ IntPtr.Zero;
pcm_data null;
pcm_time_stamp_ 10; //时间戳自增10毫秒
视频窗体数据采集 if (video_push_type_ ! (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)return;if (texture_ null || video_width_ ! Screen.width || video_height_ ! Screen.height){Debug.Log(OnPostRender screen changed scr_width: Screen.width scr_height: Screen.height);if (screen_image_ ! IntPtr.Zero){Marshal.FreeHGlobal(screen_image_);screen_image_ IntPtr.Zero;}if (texture_ ! null){UnityEngine.Object.Destroy(texture_);texture_ null;}video_width_ Screen.width;video_height_ Screen.height;texture_ new Texture2D(video_width_, video_height_, TextureFormat.BGRA32, false);screen_image_ Marshal.AllocHGlobal(video_width_ * 4 * video_height_);Debug.Log(OnPostRender screen changed--);return;}
从Texture拿到数据然后调用OnPostRGBAData()数据传递到原生接口即可。
原生接口模块初始化 /** nt_publisher_wrapper.cs* Githubhttps://github.com/daniulive/SmarterStreaming** InitSDK主要完成原生模块的初始化工作 */private bool InitSDK(){if (!is_pusher_sdk_init_){// 设置日志路径(请确保目录存在)Debug.LogError(NT_SL_SetPath);String log_path ./;NTSmartLog.NT_SL_SetPath(log_path);UInt32 isInited NTSmartPublisherSDK.NT_PB_Init(0, IntPtr.Zero);if (isInited ! 0){Debug.Log(调用NT_PB_Init失败..);return false;}is_pusher_sdk_init_ true;}return true;}
RTMP推送模块基础参数配置 private void SetCommonOptionToPublisherSDK(){if (!IsPublisherHandleAvailable()){Debug.Log(SetCommonOptionToPublisherSDK, publisher handle with null..);return;}NTSmartPublisherSDK.NT_PB_ClearLayersConfig(publisher_handle_, 0,0, IntPtr.Zero);if (video_option_ (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER){// 第0层填充RGBA矩形, 目的是保证帧率, 颜色就填充全黑int red 0;int green 0;int blue 0;int alpha 255;NT_PB_RGBARectangleLayerConfig rgba_layer_c0 new NT_PB_RGBARectangleLayerConfig();rgba_layer_c0.base_.type_ (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;rgba_layer_c0.base_.index_ 0;rgba_layer_c0.base_.enable_ 1;rgba_layer_c0.base_.region_.x_ 0;rgba_layer_c0.base_.region_.y_ 0;rgba_layer_c0.base_.region_.width_ video_width_;rgba_layer_c0.base_.region_.height_ video_height_;rgba_layer_c0.base_.offset_ Marshal.OffsetOf(rgba_layer_c0.GetType(), base_).ToInt32();rgba_layer_c0.base_.cb_size_ (uint)Marshal.SizeOf(rgba_layer_c0);rgba_layer_c0.red_ System.BitConverter.GetBytes(red)[0];rgba_layer_c0.green_ System.BitConverter.GetBytes(green)[0];rgba_layer_c0.blue_ System.BitConverter.GetBytes(blue)[0];rgba_layer_c0.alpha_ System.BitConverter.GetBytes(alpha)[0];IntPtr rgba_conf Marshal.AllocHGlobal(Marshal.SizeOf(rgba_layer_c0));Marshal.StructureToPtr(rgba_layer_c0, rgba_conf, true);UInt32 rgba_r NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,rgba_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE,0, IntPtr.Zero);Marshal.FreeHGlobal(rgba_conf);NT_PB_ExternalVideoFrameLayerConfig external_layer_c1 new NT_PB_ExternalVideoFrameLayerConfig();external_layer_c1.base_.type_ (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;external_layer_c1.base_.index_ 1;external_layer_c1.base_.enable_ 1;external_layer_c1.base_.region_.x_ 0;external_layer_c1.base_.region_.y_ 0;external_layer_c1.base_.region_.width_ video_width_;external_layer_c1.base_.region_.height_ video_height_;external_layer_c1.base_.offset_ Marshal.OffsetOf(external_layer_c1.GetType(), base_).ToInt32();external_layer_c1.base_.cb_size_ (uint)Marshal.SizeOf(external_layer_c1);IntPtr external_layer_conf Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c1));Marshal.StructureToPtr(external_layer_c1, external_layer_conf, true);UInt32 external_r NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,external_layer_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME,0, IntPtr.Zero);Marshal.FreeHGlobal(external_layer_conf);}else if (video_option_ (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA){CameraInfo camera cameras_[cur_sel_camera_index_];NT_PB_VideoCaptureCapability cap camera.capabilities_[cur_sel_camera_resolutions_index_];SetVideoCaptureDeviceBaseParameter(camera.id_.ToString(), cap.width_, cap.height_);}SetFrameRate((uint)video_fps_);Int32 type 0; //软编码Int32 encoder_id 1;UInt32 codec_id (UInt32)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;Int32 param1 0;SetVideoEncoder(type, encoder_id, codec_id, param1);SetVideoQuality(CalVideoQuality(video_width_, video_height_, is_h264_encoder_));SetVideoBitRate(CalBitRate(video_fps_, video_width_, video_height_));SetVideoMaxBitRate((CalMaxKBitRate(video_fps_, video_width_, video_height_, false)));SetVideoKeyFrameInterval((key_frame_interval_));if (is_h264_encoder_){SetVideoEncoderProfile(1);}SetVideoEncoderSpeed(CalVideoEncoderSpeed(video_width_, video_height_, is_h264_encoder_));// 音频相关设置SetAuidoInputDeviceId(0);SetPublisherAudioCodecType(1);SetPublisherMute(is_mute_);SetEchoCancellation(0, 0);SetNoiseSuppression(0);SetAGC(0);SetVAD(0);SetInputAudioVolume(Convert.ToSingle(audio_input_volume_));}
开始和停止预览 public bool StartPreview(){if(CheckPublisherHandleAvailable() false)return false;video_preview_image_callback_ new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(publisher_handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);if (NTBaseCodeDefine.NT_ERC_OK ! NTSmartPublisherSDK.NT_PB_StartPreview(publisher_handle_, 0, IntPtr.Zero)){if (0 publisher_handle_count_){NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);publisher_handle_ IntPtr.Zero;}return false;}publisher_handle_count_;is_previewing_ true;return true;}public void StopPreview(){if (is_previewing_ false) return;is_previewing_ false;publisher_handle_count_--;NTSmartPublisherSDK.NT_PB_StopPreview(publisher_handle_);if (0 publisher_handle_count_){NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);publisher_handle_ IntPtr.Zero;}}
预览数据回调 //预览数据回调public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image){NT_PB_Image pb_image (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));NT_VideoFrame pVideoFrame new NT_VideoFrame();pVideoFrame.width_ pb_image.width_;pVideoFrame.height_ pb_image.height_;pVideoFrame.stride_ pb_image.stride_[0];Int32 argb_size pb_image.stride_[0] * pb_image.height_;pVideoFrame.plane_data_ new byte[argb_size];if (argb_size 0){Marshal.Copy(pb_image.plane_[0],pVideoFrame.plane_data_,0, argb_size);}{cur_image_ pVideoFrame;}}
开始推送和停止推送 public bool StartPublisher(String url){if (CheckPublisherHandleAvailable() false) return false;if (publisher_handle_ IntPtr.Zero){return false;}if (!String.IsNullOrEmpty(url)){NTSmartPublisherSDK.NT_PB_SetURL(publisher_handle_, url, IntPtr.Zero);}if (NTBaseCodeDefine.NT_ERC_OK ! NTSmartPublisherSDK.NT_PB_StartPublisher(publisher_handle_, IntPtr.Zero)){if (0 publisher_handle_count_){NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);publisher_handle_ IntPtr.Zero;}is_publishing_ false;return false;}publisher_handle_count_;is_publishing_ true;return true;}public void StopPublisher(){if (is_publishing_ false) return;publisher_handle_count_--;NTSmartPublisherSDK.NT_PB_StopPublisher(publisher_handle_);if (0 publisher_handle_count_){NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);publisher_handle_ IntPtr.Zero;}is_publishing_ false;}
对应Unity层调用 public void btn_publish_Click(){if (publisher_wrapper_.IsPublishing()){return;}String url rtmp_pusher_url.text;if (url.Length 8){publisher_wrapper_.Close();Debug.Log(请输入RTMP推送地址);return;}publisher_wrapper_.SetVideoPushType(video_push_type_);publisher_wrapper_.SetAudioPushType(audio_push_type_);if (!publisher_wrapper_.StartPublisher(url)){Debug.LogError(调用StartPublisher失败..);return;}startPublishBtn.interactable false;stopPublishBtn.interactable !startPublishBtn.interactable;videoOptionSel.interactable false;audioOptionSel.interactable false;if (audio_push_type_ (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA|| audio_push_type_ (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER){PostUnityAudioClipData();}}public void btn_stop_publish_Click(){if (!publisher_wrapper_.IsPublishing()){return;}StopAudioSource();if (texture_ ! null){UnityEngine.Object.Destroy(texture_);texture_ null;}if (screen_image_ ! IntPtr.Zero){Marshal.FreeHGlobal(screen_image_);screen_image_ IntPtr.Zero;}publisher_wrapper_.StopPublisher();videoOptionSel.interactable true;audioOptionSel.interactable true;startPublishBtn.interactable true;stopPublishBtn.interactable !startPublishBtn.interactable;}
总结
以上是Unity环境下以采集Unity窗体和unity声音为例对接Linux平台RTMP推送整体延迟也可以达到毫秒级基本满足常用场景了感兴趣的开发者可酌情参考。
