Skip to content

Commit

Permalink
allow stream and record with differents resolutions
Browse files Browse the repository at this point in the history
  • Loading branch information
@pedroSG94
pedroSG94 committed Nov 7, 2024
1 parent 7b5a2e7 commit 59c5b00
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 9 deletions.
14 changes: 9 additions & 5 deletions app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,12 @@ class CameraFragment: Fragment(), ConnectChecker {
private lateinit var surfaceView: SurfaceView
private lateinit var bStartStop: ImageView
private lateinit var txtBitrate: TextView
private val width = 640
private val height = 480
private val vBitrate = 1200 * 1000
private val width = 1280
private val height = 720
private val vBitrate = 2000 * 1000
private val recordWidth = 1920
private val recordHeight = 1080
private val recordBitrate = 4000 * 1000
private var rotation = 0
private val sampleRate = 32000
private val isStereo = true
Expand Down Expand Up @@ -181,8 +184,9 @@ class CameraFragment: Fragment(), ConnectChecker {

private fun prepare() {
val prepared = try {
genericStream.prepareVideo(width, height, vBitrate, rotation = rotation) &&
genericStream.prepareAudio(sampleRate, isStereo, aBitrate)
genericStream.prepareVideo(width, height, vBitrate, rotation = rotation,
recordWidth = recordWidth, recordHeight = recordHeight, recordBitrate = recordBitrate
) && genericStream.prepareAudio(sampleRate, isStereo, aBitrate)
} catch (e: IllegalArgumentException) {
false
}
Expand Down
71 changes: 67 additions & 4 deletions library/src/main/java/com/pedro/library/base/StreamBase.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import com.pedro.library.util.FpsListener
import com.pedro.library.util.streamclient.StreamBaseClient
import com.pedro.library.view.GlStreamInterface
import java.nio.ByteBuffer
import kotlin.math.max


/**
Expand All @@ -74,6 +75,7 @@ abstract class StreamBase(
}
//video and audio encoders
private val videoEncoder by lazy { VideoEncoder(getVideoData) }
private val videoEncoderRecord by lazy { VideoEncoder(getVideoDataRecord) }
private val audioEncoder by lazy { AudioEncoder(getAacData) }
//video render
private val glInterface = GlStreamInterface(context)
Expand All @@ -90,6 +92,7 @@ abstract class StreamBase(
private set
var audioSource: AudioSource = aSource
private set
private var differentRecordResolution = false

/**
* Necessary only one time before start preview, stream or record.
Expand All @@ -99,23 +102,41 @@ abstract class StreamBase(
* @param level codec value from MediaCodecInfo.CodecProfileLevel class
*
* @throws IllegalArgumentException if current video parameters are not supported by the VideoSource
* @throws IllegalArgumentException if you use differentRecordResolution but the aspect ratio is not the same than stream resolution
* @return True if success, False if failed
*/
@Throws(IllegalArgumentException::class)
@JvmOverloads
fun prepareVideo(width: Int, height: Int, bitrate: Int, fps: Int = 30, iFrameInterval: Int = 2,
rotation: Int = 0, profile: Int = -1, level: Int = -1): Boolean {
rotation: Int = 0, profile: Int = -1, level: Int = -1, recordWidth: Int = width, recordHeight: Int = height, recordBitrate: Int = bitrate): Boolean {
if (isStreaming || isRecording || isOnPreview) {
throw IllegalStateException("Stream, record and preview must be stopped before prepareVideo")
}
val videoResult = videoSource.init(width, height, fps, rotation)
differentRecordResolution = false
if (recordWidth != width && recordHeight != height) {
if (recordWidth.toDouble() / recordHeight.toDouble() != width.toDouble() / height.toDouble()) {
throw IllegalArgumentException("The aspect ratio of record and stream resolution must be the same")
}
differentRecordResolution = true
}
val videoResult = videoSource.init(max(width, recordWidth), max(height, recordHeight), fps, rotation)
if (videoResult) {
if (differentRecordResolution) {
//using different record resolution
if (rotation == 90 || rotation == 270) glInterface.setEncoderRecordSize(recordHeight, recordWidth)
else glInterface.setEncoderRecordSize(recordWidth, recordHeight)
}
if (rotation == 90 || rotation == 270) glInterface.setEncoderSize(height, width)
else glInterface.setEncoderSize(width, height)
val isPortrait = rotation == 90 || rotation == 270
glInterface.setIsPortrait(isPortrait)
glInterface.setCameraOrientation(if (rotation == 0) 270 else rotation - 90)
glInterface.forceOrientation(videoSource.getOrientationConfig())
if (differentRecordResolution) {
val result = videoEncoderRecord.prepareVideoEncoder(recordWidth, recordHeight, fps, recordBitrate, rotation,
iFrameInterval, FormatVideoEncoder.SURFACE, profile, level)
if (!result) return false
}
return videoEncoder.prepareVideoEncoder(width, height, fps, bitrate, rotation,
iFrameInterval, FormatVideoEncoder.SURFACE, profile, level)
}
Expand Down Expand Up @@ -165,6 +186,9 @@ abstract class StreamBase(
if (videoEncoder.isRunning) {
videoEncoder.requestKeyframe()
}
if (videoEncoderRecord.isRunning) {
videoEncoderRecord.requestKeyframe()
}
}

/**
Expand All @@ -185,6 +209,7 @@ abstract class StreamBase(
fun forceFpsLimit(enabled: Boolean) {
val fps = if (enabled) videoEncoder.fps else 0
videoEncoder.setForceFps(fps)
videoEncoderRecord.setForceFps(fps)
glInterface.forceFpsLimit(fps)
}

Expand All @@ -194,6 +219,7 @@ abstract class StreamBase(
*/
fun forceCodecType(codecTypeVideo: CodecUtil.CodecType, codecTypeAudio: CodecUtil.CodecType) {
videoEncoder.forceCodecType(codecTypeVideo)
videoEncoderRecord.forceCodecType(codecTypeVideo)
audioEncoder.forceCodecType(codecTypeAudio)
}

Expand Down Expand Up @@ -224,7 +250,10 @@ abstract class StreamBase(
if (isRecording) throw IllegalStateException("Record already started, stopRecord before startRecord again")
recordController.startRecord(path, listener)
if (!isStreaming) startSources()
else videoEncoder.requestKeyframe()
else {
videoEncoder.requestKeyframe()
videoEncoderRecord.requestKeyframe()
}
}

/**
Expand Down Expand Up @@ -356,7 +385,15 @@ abstract class StreamBase(
fun changeVideoSource(source: VideoSource) {
val wasRunning = videoSource.isRunning()
val wasCreated = videoSource.created
if (wasCreated) source.init(videoEncoder.width, videoEncoder.height, videoEncoder.fps, videoEncoder.rotation)
if (wasCreated) {
var width = videoEncoder.width
var height = videoEncoder.height
if (differentRecordResolution) {
width = max(width, videoEncoderRecord.width)
height = max(height, videoEncoderRecord.height)
}
source.init(width, height, videoEncoder.fps, videoEncoder.rotation)
}
videoSource.stop()
videoSource.release()
if (wasRunning) source.start(glInterface.surfaceTexture)
Expand Down Expand Up @@ -387,6 +424,7 @@ abstract class StreamBase(
*/
fun setTimestampMode(timestampModeVideo: TimestampMode, timestampModeAudio: TimestampMode) {
videoEncoder.setTimestampMode(timestampModeVideo)
videoEncoderRecord.setTimestampMode(timestampModeVideo)
audioEncoder.setTimestampMode(timestampModeAudio)
}

Expand All @@ -396,6 +434,7 @@ abstract class StreamBase(
*/
fun setEncoderErrorCallback(encoderErrorCallback: EncoderErrorCallback?) {
videoEncoder.setEncoderErrorCallback(encoderErrorCallback)
videoEncoderRecord.setEncoderErrorCallback(encoderErrorCallback)
audioEncoder.setEncoderErrorCallback(encoderErrorCallback)
}

Expand Down Expand Up @@ -453,16 +492,20 @@ abstract class StreamBase(
audioSource.start(getMicrophoneData)
val startTs = System.nanoTime() / 1000
videoEncoder.start(startTs)
if (differentRecordResolution) videoEncoderRecord.start(startTs)
audioEncoder.start(startTs)
glInterface.addMediaCodecSurface(videoEncoder.inputSurface)
if (differentRecordResolution) glInterface.addMediaCodecRecordSurface(videoEncoderRecord.inputSurface)
}

private fun stopSources() {
if (!isOnPreview) videoSource.stop()
audioSource.stop()
glInterface.removeMediaCodecSurface()
glInterface.removeMediaCodecRecordSurface()
if (!isOnPreview) glInterface.stop()
videoEncoder.stop()
videoEncoderRecord.stop()
audioEncoder.stop()
if (!isRecording) recordController.resetFormats()
}
Expand All @@ -486,6 +529,12 @@ abstract class StreamBase(
* @return true if success, false if failed
*/
fun resetVideoEncoder(): Boolean {
if (differentRecordResolution) {
glInterface.removeMediaCodecRecordSurface()
val result = videoEncoderRecord.reset()
if (!result) return false
glInterface.addMediaCodecRecordSurface(videoEncoderRecord.inputSurface)
}
glInterface.removeMediaCodecSurface()
val result = videoEncoder.reset()
if (!result) return false
Expand Down Expand Up @@ -523,6 +572,19 @@ abstract class StreamBase(
override fun getVideoData(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) {
fpsListener.calculateFps()
getVideoDataImp(videoBuffer, info)
if (!videoEncoderRecord.isRunning) recordController.recordVideo(videoBuffer, info)
}

override fun onVideoFormat(mediaFormat: MediaFormat) {
if (!videoEncoderRecord.isRunning) recordController.setVideoFormat(mediaFormat)
}
}

private val getVideoDataRecord: GetVideoData = object : GetVideoData {
override fun onVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) {
}

override fun getVideoData(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) {
recordController.recordVideo(videoBuffer, info)
}

Expand Down Expand Up @@ -553,6 +615,7 @@ abstract class StreamBase(
VideoCodec.AV1 -> CodecUtil.AV1_MIME
}
videoEncoder.type = type
videoEncoderRecord.type = type
}

/**
Expand Down
34 changes: 34 additions & 0 deletions library/src/main/java/com/pedro/library/view/GlStreamInterface.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,14 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
private val running = AtomicBoolean(false)
private val surfaceManager = SurfaceManager()
private val surfaceManagerEncoder = SurfaceManager()
private val surfaceManagerEncoderRecord = SurfaceManager()
private val surfaceManagerPhoto = SurfaceManager()
private val surfaceManagerPreview = SurfaceManager()
private val mainRender = MainRender()
private var encoderWidth = 0
private var encoderHeight = 0
private var encoderRecordWidth = 0
private var encoderRecordHeight = 0
private var streamOrientation = 0
private var previewWidth = 0
private var previewHeight = 0
Expand Down Expand Up @@ -90,6 +93,11 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
encoderHeight = height
}

fun setEncoderRecordSize(width: Int, height: Int) {
encoderRecordWidth = width
encoderRecordHeight = height
}

override fun getEncoderSize(): Point {
return Point(encoderWidth, encoderHeight)
}
Expand Down Expand Up @@ -138,6 +146,22 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
}
}

fun addMediaCodecRecordSurface(surface: Surface) {
executor?.secureSubmit {
if (surfaceManager.isReady) {
surfaceManagerEncoderRecord.release()
surfaceManagerEncoderRecord.eglSetup(surface, surfaceManager)
}
}
}

fun removeMediaCodecRecordSurface() {
threadQueue.clear()
executor?.secureSubmit {
surfaceManagerEncoderRecord.release()
}
}

override fun takePhoto(takePhotoCallback: TakePhotoCallback?) {
this.takePhotoCallback = takePhotoCallback
}
Expand Down Expand Up @@ -166,6 +190,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
sensorRotationManager.stop()
surfaceManagerPhoto.release()
surfaceManagerEncoder.release()
surfaceManagerEncoderRecord.release()
surfaceManager.release()
mainRender.release()
}
Expand Down Expand Up @@ -209,6 +234,15 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
isStreamVerticalFlip, isStreamHorizontalFlip)
surfaceManagerEncoder.swapBuffer()
}
// render VideoEncoder (record if the resolution is different than stream)
if (surfaceManagerEncoderRecord.isReady && mainRender.isReady() && !limitFps) {
val w = if (muteVideo) 0 else encoderRecordWidth
val h = if (muteVideo) 0 else encoderRecordHeight
surfaceManagerEncoderRecord.makeCurrent()
mainRender.drawScreenEncoder(w, h, orientation, streamOrientation,
isStreamVerticalFlip, isStreamHorizontalFlip)
surfaceManagerEncoderRecord.swapBuffer()
}
//render surface photo if request photo
if (takePhotoCallback != null && surfaceManagerPhoto.isReady && mainRender.isReady()) {
surfaceManagerPhoto.makeCurrent()
Expand Down

0 comments on commit 59c5b00

Please sign in to comment.