function H264Session() { let rtpTimeStamp = 0; let size1M = 1048576; //1024 * 1024 let inputBuffer = new Uint8Array(size1M); let spsSegment = null; let ppsSegment = null; let SPSParser = null; let width = 0; let height = 0; let inputLength = 0; let initalSegmentFlag = true; //用于确定是否是initSegment let initalMediaFrameFlag = true; let frameRate = null; //根据SDP或者SPS设置 let preSample = null; //上一个Sample let durationTimeCount = 0; let frameCount = 0; let inputSegBufferSub = null; //MSE使用的数据以及相关配置,顺序codecInfo -> initSegmentData -> mediaSample -> frameData //时间戳用于绘制人脸框 let decodedData = { frameData: null, //视频数据 timeStamp: null, //时间戳 initSegmentData: null, //MP4配置,用于initsegment mediaSample: null, //使用duration控制每一帧的播放时间 codecInfo: "", //MSE init时传入,用于创建mediasource }; let FRAMETYPE = { 1: 'P', 5: 'I', 6: 'SEI', 7: 'I' }; let frameType = ''; let decodeMode = 'video'; let outputSize = 0; let curSize = 0; const PREFIX = new Uint8Array(['0x00', '0x00', '0x00', '0x01']); let firstIframe = false; let SEIInfo = { ivs: null, timestamp: null, }; let preWidth = null, preHeight = null; let resetTimeCount = 0; let lastTimeStamp = 0; //const RESETTIME = 162000000; const RESETTIME = 4320000; let SEIBuffer = null; let lastSEITime = 0; function constructor() { } constructor.prototype = { init() { SPSParser = new H264SPSParser(); this.resolutionChangedCallback = () => { }; }, remuxRTPData(rtspInterleaved, rtpHeader, rtpPayload) { //console.log(rtspInterleaved) //console.log(rtpHeader) let PaddingSize = 0; let extensionHeaderLen = 0; //如果RtpHeader.X=1,则在RTP报头后跟有一个扩展报头 let PAYLOAD = null; //console.log(rtpHeader) //console.log(rtspInterleaved, rtpHeader, rtpPayload.subarray(0,5)) let RtpHeader = { V: rtpHeader[0] >>> 6, P: rtpHeader[0] & 0x20, X: rtpHeader[0] & 0x10, CC: rtpHeader[0] & 0x0F, M: (rtpHeader[1] & 0x80) >> 7, PT: rtpHeader[1] & 127, SN: (rtpHeader[2] << 8) + rtpHeader[3], timeStamp: (rtpHeader[4] << 24) + (rtpHeader[5] << 16) + (rtpHeader[6] << 8) + rtpHeader[7], SSRC: (rtpHeader[8] << 24) + (rtpHeader[9] << 16) + (rtpHeader[10] << 8) + rtpHeader[11], }; if (RtpHeader.P) { //填充 PaddingSize = rtpPayload[rtpPayload.length - 1]; console.log("Padding - " + PaddingSize); } if (RtpHeader.X) { //扩展 extensionHeaderLen = (((rtpPayload[2] << 8) | rtpPayload[3]) * 4) + 4; console.log('X: ' + rtpPayload[0]) } //console.log('extensionHeaderLen: '+ extensionHeaderLen) PAYLOAD = rtpPayload.subarray(extensionHeaderLen, rtpPayload.length - PaddingSize); rtpTimeStamp = RtpHeader.timeStamp; //console.log(rtpTimeStamp, rtpHeader[4], rtpHeader[5], rtpHeader[6] , rtpHeader[7], PAYLOAD[0] & 0x1f) /* 载荷结构(https://blog.csdn.net/davebobo/article/details/52994596) +---------------+ |0|1|2|3|4|5|6|7| +-+-+-+-+-+-+-+-+ |F|NRI| Type | +---------------+ Type = 1-23 单个NAL单元包 Type = 24,25, 26, 27聚合包 Type = 28,29, 分片单元 */ //console.log(rtspInterleaved,rtpHeader, PAYLOAD[0]); let nalType = (PAYLOAD[0] & 0x1f); //console.log(PAYLOAD[0] + ' nalType: ' + nalType); //console.log('rtpPayload.length: ' + rtpPayload.length) //console.log(nalType, PAYLOAD[0]) //console.log('nalType: ' + nalType, RtpHeader.M) switch (nalType) { case 6: //SEI //console.log(PAYLOAD, String.fromCharCode.apply(null, PAYLOAD)) if (SEIParse(PAYLOAD) === null) { return; } inputBuffer = setBuffer(inputBuffer, PREFIX); inputBuffer = setBuffer(inputBuffer, PAYLOAD); break; case 7: //SPS //console.log('SPS'); SPSParser.parse(removeH264or5EmulationBytes(PAYLOAD)); let sizeInfo = SPSParser.getSizeInfo(); //console.log(SPSParser.getSpsMap()) width = sizeInfo.width; height = sizeInfo.height; if (preWidth !== width || preHeight !== height) { console.log('resolution changed!'); console.log('preWidth: ', preWidth, ' preHeight: ', preHeight, ' width: ', width, ' height: ', height); preWidth = width; preHeight = height; } inputBuffer = setBuffer(inputBuffer, PREFIX); inputBuffer = setBuffer(inputBuffer, PAYLOAD); spsSegment = PAYLOAD; //console.log('width: ',width, 'height: ', height) curSize = sizeInfo.decodeSize; firstIframe = true; //console.log(spsSegment) if (frameRate === null) { frameRate = SPSParser.getFPS(); } break; case 8: //PPS //console.log('PPS') inputBuffer = setBuffer(inputBuffer, PREFIX); inputBuffer = setBuffer(inputBuffer, PAYLOAD); ppsSegment = PAYLOAD; //console.log(ppsSegment) break; case 28: //FU //console.log('FU'); let startBit = ((PAYLOAD[1] & 0x80) === 0x80), endBit = ((PAYLOAD[1] & 0x40) === 0x40), fuType = PAYLOAD[1] & 0x1f, payloadStartIndex = 2; //console.log('startBit: ' + startBit + ' endBit: ' + endBit) //console.log('fuType: ' + fuType) if (startBit === true && endBit === false) { let newNalHeader = new Uint8Array(1); newNalHeader[0] = ((PAYLOAD[0] & 0xe0) | fuType); //console.log('newNalHeader: ', newNalHeader[0]) //console.log('fuType: ' + fuType) //console.log((PAYLOAD[2] << 8) + PAYLOAD[3]) //console.log(new Uint8Array(PAYLOAD.subarray(0, 100))); if (false) { //赛兰摄像头,SPS,PPS,I帧打在一个RTP包中 PAYLOAD[1] = newNalHeader[0]; SPSParser.parse(removeH264or5EmulationBytes(PAYLOAD.subarray(1, 27))); let sizeInfo = SPSParser.getSizeInfo(); //console.log(sizeInfo, SPSParser.getSpsMap()) //SPS width = sizeInfo.width; height = sizeInfo.height; inputBuffer = setBuffer(inputBuffer, PREFIX); inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(1, 27)); spsSegment = PAYLOAD.subarray(1, 27); //console.log('width: ',width, 'height: ', height) curSize = sizeInfo.decodeSize; firstIframe = true; //PPS inputBuffer = setBuffer(inputBuffer, PREFIX); inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(30, 34)); ppsSegment = PAYLOAD.subarray(30, 34); //I inputBuffer = setBuffer(inputBuffer, PREFIX); inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(38, PAYLOAD.length)); } else { //console.log(newNalHeader[0] & 0x1f) inputBuffer = setBuffer(inputBuffer, PREFIX); inputBuffer = setBuffer(inputBuffer, newNalHeader); inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(payloadStartIndex, PAYLOAD.length)); } } else { //console.log(startBit, endBit, 'endBit') inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(payloadStartIndex, PAYLOAD.length)); } //console.log(startBit,endBit) // if(endBit === true) { // end = true; // } break; case 1: inputBuffer = setBuffer(inputBuffer, PREFIX); inputBuffer = setBuffer(inputBuffer, PAYLOAD); break; default: //console.log('nalType: ' + nalType); //console.log(PAYLOAD) break; } //console.log('M: ' + RtpHeader.M + ' ' + (rtpHeader[1] & 0x80)) //check marker bit if (RtpHeader.M) { if (!firstIframe) { inputLength = 0; return; } let inputBufferSub = inputBuffer.subarray(0, inputLength); frameType = FRAMETYPE[inputBufferSub[4] & 0x1f]; //只根据视频帧计算resetTimeCount if (frameType !== 'SEI') { // rtp时间戳周期为RESETTIME,如果单向递增,设为0 if (lastTimeStamp - rtpTimeStamp > (RESETTIME / 2)) { //判断lastTimeStamp远大于rtpTimeStamp,防止后一帧比前一帧先到的情况 //console.log(lastTimeStamp - rtpTimeStamp) console.warn('时间戳重置', lastTimeStamp, rtpTimeStamp, frameType, new Date()) resetTimeCount++; } rtpTimeStamp = rtpTimeStamp + RESETTIME * resetTimeCount; } else { //同一帧的SEI比视频发送慢时 if (rtpTimeStamp - lastTimeStamp > (RESETTIME / 2)) { console.warn('SEI翻转', rtpTimeStamp, lastTimeStamp); rtpTimeStamp = rtpTimeStamp + RESETTIME * (resetTimeCount - 1); } else { rtpTimeStamp = rtpTimeStamp + RESETTIME * resetTimeCount; } //同一帧的SEI比视频发送快时 // if(rtpTimeStamp > lastTimeStamp) { // rtpTimeStamp = rtpTimeStamp + RESETTIME * resetTimeCount; // } else { // rtpTimeStamp = rtpTimeStamp + RESETTIME * (resetTimeCount + 1); // } } //console.log('frameType: ', frameType, 'rtpTimeStamp: ', rtpTimeStamp) if (frameType === 'SEI') { //SEI被分片(nal === 28)时,分片包发送完后marker为1,不会和视频帧一起 SEIBuffer = inputBuffer.subarray(4, inputLength); //console.log(SEIBuffer) inputBufferSub = new Uint8Array(); } if (SEIBuffer) { let SEI = SEIParse(SEIBuffer); if (SEI) { SEIInfo.ivs = SEI; SEIInfo.timestamp = rtpTimeStamp; decodedData.SEIInfo = SEIInfo; if ((rtpTimeStamp - lastSEITime) !== (90000 / frameRate)) { //console.log('SEI 时间差:', (rtpTimeStamp - lastTime), rtpTimeStamp, lastTime) } lastSEITime = rtpTimeStamp; SEIInfo = { ivs: null, timestamp: 0, }; } SEIBuffer = null; } if (!initalSegmentFlag) { decodedData.initSegmentData = null; decodedData.codecInfo = null; } else { initalSegmentFlag = false; const info = { id: 1, width: width, height: height, type: "video", profileIdc: SPSParser.getSpsValue("profile_idc"), profileCompatibility: 0, levelIdc: SPSParser.getSpsValue("level_idc"), sps: [spsSegment], pps: [ppsSegment], timescale: 1e3, fps: frameRate }; decodedData.initSegmentData = info; decodedData.codecInfo = SPSParser.getCodecInfo(); //console.log(info.pps) } if (frameType === 'I') { //console.log('ppsSegment: ', ppsSegment) let h264parameterLength = spsSegment.length + ppsSegment.length + 8; inputSegBufferSub = inputBufferSub.subarray(h264parameterLength, inputBufferSub.length); } else { inputSegBufferSub = inputBufferSub.subarray(0, inputBufferSub.length); } if (inputSegBufferSub.length) { let segSize = inputSegBufferSub.length - 4; //mp4 box头 inputSegBufferSub[0] = (segSize & 0xFF000000) >>> 24; inputSegBufferSub[1] = (segSize & 0xFF0000) >>> 16; inputSegBufferSub[2] = (segSize & 0xFF00) >>> 8; inputSegBufferSub[3] = (segSize & 0xFF); decodedData.frameData = new Uint8Array(inputSegBufferSub); let sample = { duration: Math.round((1 / frameRate) * 1000), size: inputSegBufferSub.length, frame_time_stamp: null, frameDuration: null, }; sample.frame_time_stamp = rtpTimeStamp; //Todo:暂时为null,通过帧率控制duration if (initalMediaFrameFlag) { sample.frameDuration = 0; initalMediaFrameFlag = false; } else { if (frameRate) { frameCount++; if (!(frameCount % frameRate)) { //每秒最后一帧时 sample.frameDuration = 1000 - durationTimeCount; frameCount = 0; durationTimeCount = 0; } else { sample.frameDuration = Math.round(1000 / frameRate); durationTimeCount += Math.round(1000 / frameRate); } } else { sample.frameDuration = (sample.frame_time_stamp - preSample.frame_time_stamp) / 90; // 时钟频率90000,timescale=1000 } //console.log(sample.frameDuration) } preSample = sample; decodedData.mediaSample = sample; decodedData.timeStamp = rtpTimeStamp; } this.handleDecodedData(decodedData); inputLength = 0; decodedData.SEIInfo = null; inputSegBufferSub = null; if (frameType !== 'SEI') { lastTimeStamp = RtpHeader.timeStamp; } frameType = ''; } //console.log('xxxxxxxxxxxxxxxxxxxxxxxxx') }, set rtpSessionCallback(func) { this.handleDecodedData = func; }, setFrameRate(fps) { frameRate = fps; //console.log('frameRate: ', frameRate) }, setResolutionChangedCallback(callback) { this.resolutionChangedCallback = callback; } } return new constructor(); function setBuffer(buffer1, buffer2) { let bufferTemp = buffer1; if ((inputLength + buffer2.length) > buffer1.length) { bufferTemp = new Uint8Array(buffer1.length + size1M); } bufferTemp.set(buffer2, inputLength); inputLength += buffer2.length; return bufferTemp; } } /** * 去除SPS中的Emulation字节 * @param data SPS源数据 * @returns {Array} 去除后Emulation字节后的SPS */ function removeH264or5EmulationBytes(data) { let toSize = 0; let i = 0; let to = []; let dataLength = data.length; while (i < dataLength) { if (i + 2 < dataLength && data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 3) { to[toSize] = to[toSize + 1] = 0; toSize += 2; i += 3; } else { to[toSize] = data[i]; toSize += 1; i += 1; } } return to; } /** * 解析SEI信息 * @param data * @return {Array} */ function SEIParse(data) { //console.log(data) if ((data[0] & 0x1f) !== 6) { //非SEI return null; } if (data[1] !== 0x55 || data[2] !== 0x4C || data[3] !== 0x53 || data[4] !== 0x40) { //ULS@开头 console.warn('unknown SEI type'); return null; } let dataLength = data.length; let type = (data[5] << 8) + data[6]; let checkSum = (data[7] << 8) + data[8]; // if(dataLength !== (checkSum + 1 + 4 + 4)) { // console.log('SEI check fail!'); // return null; // } let result; data = data.subarray(9); switch (type) { case 0: result = parseFace(data); break; case 1: //console.log(parseBody(data)) result = parseBody(data); break; case 2: result = parseRegion(data); break; case 3: result = parseBodyEx(data); break; case 4: result = parseOverSpeed(data); break; default: result = null; break; } return result; } function parseFace(data) { //console.log(data) let dataLength = data.length; let contents = []; while (dataLength > 0) { //console.log('dataLength: ', dataLength) let x0 = ((data[4] & 0x7f) << 8) + data[5], y0 = ((data[6] & 0x7f) << 8) + data[7], width = ((data[8] & 0x7f) << 8) + data[9] - x0, height = ((data[10] & 0x7f) << 8) + data[11] - y0; let content = { type: 'rect', id: (data[2] << 8) + data[3], rect: [x0, y0, width, height], state: data[1] & 0x01, quality: (data[1] & 0x02) >> 1, }; Array.prototype.push.apply(contents, [content]); data = data.subarray(12); dataLength = data.length; } return contents; } function parseBody(data) { let dataLength = data.length; let contents = []; while (dataLength > 0) { //console.log('dataLength: ', dataLength) let x0 = ((data[4] & 0x7f) << 8) + data[5], y0 = ((data[6] & 0x7f) << 8) + data[7], width = ((data[8] & 0x7f) << 8) + data[9] - x0, height = ((data[10] & 0x7f) << 8) + data[11] - y0, boxConfidence = ((data[12] & 0x7f) << 8) + data[13]; let points = []; for (let i = 0; i < 17; i++) { let point = { x: ((data[16 + i * 8] & 0x7f) << 8) + data[17 + i * 8], y: ((data[18 + i * 8] & 0x7f) << 8) + data[19 + i * 8], confidence: ((data[20 + i * 8] & 0x7f) << 8) + data[21 + i * 8], // x: Math.random() * 8191, // y: Math.random() * 8191, // confidence: 1, }; points.push(point); } let content = { type: 'coco-pose', id: (data[2] << 8) + data[3], handsUp: data[1] & 0x04, boundingBox: [x0, y0, width, height], boxConfidence, points: parseBodyToTree(points), state: data[1] & 0x01, }; Array.prototype.push.apply(contents, [content]); data = data.subarray(152); dataLength = data.length; } return contents; } function parseRegion(data) { let dataLength = data.length; let contents = []; while (dataLength > 0) { let pointNum = (data[0] << 8) + data[1], state = data[3] & 0x03, area = []; for (let i = 0; i < pointNum; i++) { let point = { x: ((data[i * 4 + 4] & 0x7f) << 8) + data[i * 4 + 5], y: ((data[i * 4 + 6] & 0x7f) << 8) + data[i * 4 + 7], }; area.push(point); } let content = { type: 'region-detect', state, area }; Array.prototype.push.apply(contents, [content]); data = data.subarray(pointNum * 4 + 4); dataLength = data.length; } return contents; } function parseBodyEx(data) { let dataLength = data.length; let contents = []; while (dataLength > 0) { //console.log('dataLength: ', dataLength) let x0 = ((data[4] & 0x7f) << 8) + data[5], y0 = ((data[6] & 0x7f) << 8) + data[7], width = ((data[8] & 0x7f) << 8) + data[9] - x0, height = ((data[10] & 0x7f) << 8) + data[11] - y0, boxConfidence = ((data[12] & 0x7f) << 8) + data[13]; let points = []; for (let i = 0; i < 17; i++) { let point = { x: ((data[16 + i * 8] & 0x7f) << 8) + data[17 + i * 8], y: ((data[18 + i * 8] & 0x7f) << 8) + data[19 + i * 8], confidence: ((data[20 + i * 8] & 0x7f) << 8) + data[21 + i * 8], }; points.push(point); } let guides = [ { x: ((data[16 + 17 * 8] & 0x7f) << 8) + data[17 + 17 * 8], y: ((data[18 + 17 * 8] & 0x7f) << 8) + data[19 + 17 * 8], }, { x: ((data[20 + 17 * 8] & 0x7f) << 8) + data[21 + 17 * 8], y: ((data[22 + 17 * 8] & 0x7f) << 8) + data[23 + 17 * 8], } ]; let content = { type: 'coco-poseex', id: (data[2] << 8) + data[3], Loiter: data[1] & 0x03, Standing: (data[1] & 0x04) >> 2, Alone: (data[1] & 0x08) >> 3, boundingBox: [x0, y0, width, height], boxConfidence, points: parseBodyToTree(points), guides, state: 1 }; Array.prototype.push.apply(contents, [content]); data = data.subarray(160); dataLength = data.length; } return contents; } function parseOverSpeed(data) { let dataLength = data.length; let contents = []; while (dataLength > 0) { //console.log('dataLength: ', dataLength) let speed = ((data[4] & 0x7f) << 8) + data[5], overSpeed = data[1] & 0x01; let content = { type: 'over-speed', id: (data[2] << 8) + data[3], speed, overSpeed, state: 1, }; Array.prototype.push.apply(contents, [content]); data = data.subarray(8); dataLength = data.length; } return contents; } /** * 将智能帧中的人体姿态点转化为树结构(双亲表示法) * @param points */ function parseBodyToTree(points) { let newPoints = []; newPoints[0] = {...points[0], parent: -1, pointColor: '#FF0002', lineColor: '#FF0002'}; newPoints[1] = {...points[1], parent: 0, pointColor: '#FF0002', lineColor: '#FF0002'}; newPoints[2] = {...points[2], parent: 0, pointColor: '#FF0002', lineColor: '#FF0002'}; newPoints[3] = {...points[5], parent: 17, pointColor: '#D9E34F', lineColor: '#FF0002'}; newPoints[4] = {...points[6], parent: 17, pointColor: '#D9E34F', lineColor: '#FF0002'}; newPoints[5] = {...points[3], parent: 1, pointColor: '#00FF00', lineColor: '#FF0002'}; newPoints[6] = {...points[4], parent: 2, pointColor: '#FFAC00', lineColor: '#FF0002'}; newPoints[7] = {...points[7], parent: 3, pointColor: '#00FF45', lineColor: '#'}; newPoints[8] = {...points[11], parent: 17, pointColor: '#EEC446', lineColor: '#'}; newPoints[9] = {...points[8], parent: 4, pointColor: '#43D3AF', lineColor: '#'}; newPoints[10] = {...points[12], parent: 17, pointColor: '#7A93E8', lineColor: '#'}; newPoints[11] = {...points[9], parent: 7, pointColor: '#EFB842', lineColor: '#'}; newPoints[12] = {...points[13], parent: 8, pointColor: '#E56C00', lineColor: '#'}; newPoints[13] = {...points[10], parent: 9, pointColor: '#47CD43', lineColor: '#0096FF'}; newPoints[14] = {...points[14], parent: 10, pointColor: '#3868D2', lineColor: '#00FF51'}; newPoints[15] = {...points[15], parent: 12, pointColor: '#DF4D01', lineColor: '#5100FF'}; newPoints[16] = {...points[16], parent: 14, pointColor: '#1E48D4', lineColor: '#00FFA0'}; newPoints[17] = { x: (points[5].x + points[6].x) / 2, y: (points[5].y + points[6].y) / 2, parent: -1, pointColor: '#D9E34F', lineColor: '#00FFA0' }; return newPoints; }