All/HTML/wine-worker/public/static/h265/src/H264Session.js

function H264Session() {
    let rtpTimeStamp = 0;
    let size1M = 1048576; //1024 * 1024
    let inputBuffer = new Uint8Array(size1M);
    let spsSegment = null;
    let ppsSegment = null;

    let SPSParser = null;

    let width = 0;
    let height = 0;
    let inputLength = 0;

    let initalSegmentFlag = true; //用于确定是否是initSegment
    let initalMediaFrameFlag = true;

    let frameRate = null; //根据SDP或者SPS设置
    let preSample = null; //上一个Sample

    let inputSegBufferSub = null;

    //MSE使用的数据以及相关配置，顺序codecInfo -> initSegmentData -> mediaSample -> frameData
    //时间戳用于绘制人脸框
    let decodedData = {
        frameData: null, //视频数据
        timeStamp: null, //时间戳
        initSegmentData: null, //MP4配置,用于initsegment
        mediaSample: null, //使用duration控制每一帧的播放时间
        codecInfo: "", //MSE init时传入，用于创建mediasource
    };
    let FRAMETYPE = {
        1: 'P',
        5: 'I',
        6: 'SEI',
        7: 'I'
    };
    let frameType = '';
    let decodeMode = 'canvas';
    let outputSize = 0;
    let curSize = 0;

    const PREFIX = new Uint8Array(['0x00', '0x00', '0x00', '0x01']);

    let firstIframe = false;

    let SEIInfo = {
        ivs: null,
        timestamp:null,
    };

    let preWidth = null,
        preHeight = null;
    let resetTimeCount = 0;
    let lastTimeStamp = 0;
    //const RESETTIME = 162000000;
    const RESETTIME = 4320000;
    let SEIBuffer = null;
    let lastTime =0;
    let decoder = null;
    function constructor() {

    }

    constructor.prototype = {
        init() {
            SPSParser = new H264SPSParser();
            decoder = new H264Decoder();
            this.resolutionChangedCallback = ()=>{};
        },

        remuxRTPData(rtspInterleaved, rtpHeader, rtpPayload) {
            //console.log(rtspInterleaved)
            //console.log(rtpHeader)
            let PaddingSize = 0;
            let extensionHeaderLen = 0; //如果RtpHeader.X=1，则在RTP报头后跟有一个扩展报头
            let PAYLOAD = null;
//console.log(rtpHeader)
//console.log(rtspInterleaved, rtpHeader, rtpPayload.subarray(0,5))
            let RtpHeader = {
                V: rtpHeader[0] >>> 6,
                P: rtpHeader[0] & 0x20,
                X: rtpHeader[0] & 0x10,
                CC: rtpHeader[0] & 0x0F,
                M: (rtpHeader[1] & 0x80) >> 7,
                PT: rtpHeader[1] & 127,
                SN: (rtpHeader[2] << 8) + rtpHeader[3],
                timeStamp: (rtpHeader[4] << 24) + (rtpHeader[5] << 16) + (rtpHeader[6] << 8) + rtpHeader[7],
                SSRC: (rtpHeader[8] << 24) + (rtpHeader[9] << 16) + (rtpHeader[10] << 8) + rtpHeader[11],
            };
            if (RtpHeader.P) { //填充
                PaddingSize = rtpPayload[rtpPayload.length - 1];
                //console.log("Padding - " + PaddingSize);
            }

            if (RtpHeader.X) { //扩展
                extensionHeaderLen = (((rtpPayload[2] << 8) | rtpPayload[3]) * 4) + 4;
                console.log('X: ' + rtpPayload[0])
            }
//console.log('extensionHeaderLen: '+ extensionHeaderLen)
            PAYLOAD = rtpPayload.subarray(extensionHeaderLen, rtpPayload.length - PaddingSize);
            rtpTimeStamp = RtpHeader.timeStamp;

            //console.log(rtpTimeStamp, rtpHeader[4], rtpHeader[5], rtpHeader[6] , rtpHeader[7], PAYLOAD[0] & 0x1f)
            /* 载荷结构(https://blog.csdn.net/davebobo/article/details/52994596)
            +---------------+
            |0|1|2|3|4|5|6|7|
            +-+-+-+-+-+-+-+-+
            |F|NRI|  Type   |
            +---------------+
            Type = 1-23 单个NAL单元包
            Type = 24,25, 26, 27聚合包
            Type = 28，29, 分片单元
            */
            //console.log(rtspInterleaved,rtpHeader, PAYLOAD[0]);
            let nalType = (PAYLOAD[0] & 0x1f);
            //console.log(PAYLOAD[0] + ' nalType: ' + nalType);

//console.log('rtpPayload.length: ' + rtpPayload.length)
//console.log(nalType, PAYLOAD[0])
//console.log('nalType: ' + nalType, RtpHeader.M, rtpTimeStamp)
            //console.log('nalType: ', nalType, ' timestamp: ', rtpTimeStamp);
            switch (nalType) {
                case 6: //SEI
                    //console.log(PAYLOAD, String.fromCharCode.apply(null, PAYLOAD))
                    if(SEIParse(PAYLOAD) === null) {
                        return;
                    }
                    inputBuffer = setBuffer(inputBuffer, PREFIX);
                    inputBuffer = setBuffer(inputBuffer, PAYLOAD);
                    break;
                case 7: //SPS
                    //console.log('SPS');
                    SPSParser.parse(removeH264or5EmulationBytes(PAYLOAD));
                    let sizeInfo = SPSParser.getSizeInfo();
                    //console.log(SPSParser.getSpsMap())
                    width = sizeInfo.width;
                    height = sizeInfo.height;

                    if(preWidth !== width || preHeight !== height) {
                        console.log('resolution changed!');
                        console.log('preWidth: ', preWidth, ' preHeight: ', preHeight, ' width: ', width, ' height: ', height);
                        preWidth = width;
                        preHeight = height;
                    }
                    inputBuffer = setBuffer(inputBuffer, PREFIX);
                    inputBuffer = setBuffer(inputBuffer, PAYLOAD);
                    spsSegment = PAYLOAD;
                    //console.log('width： ',width, 'height: ', height)
                    curSize = sizeInfo.decodeSize;
                    firstIframe = true;
//console.log(spsSegment)
                    if (frameRate === null) {
                        frameRate = SPSParser.getFPS();
                    }
                    break;
                case 8: //PPS
                    //console.log('PPS')
                    inputBuffer = setBuffer(inputBuffer, PREFIX);
                    inputBuffer = setBuffer(inputBuffer, PAYLOAD);
                    ppsSegment = PAYLOAD;
//console.log(ppsSegment)
                    break;
                case 28: //FU
                    //console.log('FU');
                    let startBit = ((PAYLOAD[1] & 0x80) === 0x80),
                        endBit = ((PAYLOAD[1] & 0x40) === 0x40),
                        fuType = PAYLOAD[1] & 0x1f,
                        payloadStartIndex = 2;
                    //console.log('startBit: ' + startBit + ' endBit: ' + endBit)
                    //console.log('fuType: ' + fuType)
                    if (startBit === true && endBit === false) {
                        let newNalHeader = new Uint8Array(1);
                        newNalHeader[0] = ((PAYLOAD[0] & 0xe0) | fuType);
                        //console.log('newNalHeader： ', newNalHeader[0])
                        //console.log('fuType: ' + fuType)
                        //console.log((PAYLOAD[2] << 8) + PAYLOAD[3])
//console.log(new Uint8Array(PAYLOAD.subarray(0, 100)));
                        if (false) { //赛兰摄像头，SPS,PPS,I帧打在一个RTP包中
                            PAYLOAD[1] = newNalHeader[0];
                            SPSParser.parse(removeH264or5EmulationBytes(PAYLOAD.subarray(1, 27)));
                            let sizeInfo = SPSParser.getSizeInfo();
//console.log(sizeInfo, SPSParser.getSpsMap())
                            //SPS
                            width = sizeInfo.width;
                            height = sizeInfo.height;
                            inputBuffer = setBuffer(inputBuffer, PREFIX);
                            inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(1, 27));
                            spsSegment = PAYLOAD.subarray(1, 27);
                            //console.log('width： ',width, 'height: ', height)
                            curSize = sizeInfo.decodeSize;
                            firstIframe = true;

                            //PPS
                            inputBuffer = setBuffer(inputBuffer, PREFIX);
                            inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(30, 34));
                            ppsSegment = PAYLOAD.subarray(30, 34);

                            //I
                            inputBuffer = setBuffer(inputBuffer, PREFIX);
                            inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(38, PAYLOAD.length));

                        } else {
                            //console.log(newNalHeader[0] & 0x1f)
                            inputBuffer = setBuffer(inputBuffer, PREFIX);
                            inputBuffer = setBuffer(inputBuffer, newNalHeader);
                            inputBuffer = setBuffer(inputBuffer, PAYLOAD.subarray(payloadStartIndex, PAYLOAD.length));
                        }

                    } else {
                        //console.log(startBit, endBit, 'endBit')
                        inputBuffer = setBuffer(inputBuffer,
                            PAYLOAD.subarray(payloadStartIndex, PAYLOAD.length));
                    }
//console.log(startBit,endBit)
                    // if(endBit === true) {
                    //     end = true;
                    // }
                    break;
                case 1:
                    inputBuffer = setBuffer(inputBuffer, PREFIX);
                    inputBuffer = setBuffer(inputBuffer, PAYLOAD);
                    break;
                default:
                    //console.log('nalType: ' + nalType);
                    //console.log(PAYLOAD)
                    break;
            }

            //console.log('ppsSegment: ', ppsSegment)
            //console.log('nalType: ' + nalType);
//console.log('M: ' + RtpHeader.M + ' ' + (rtpHeader[1] & 0x80))
            //console.log(RtpHeader)


//console.log('RtpHeader.M: ', RtpHeader.M)
            //check marker bit
            if (RtpHeader.M) {
                if (!firstIframe) {
                    inputLength = 0;
                    return;
                }
                let inputBufferSub = inputBuffer.subarray(0, inputLength);
                frameType = FRAMETYPE[inputBufferSub[4] & 0x1f];

                //只根据视频帧计算resetTimeCount
                if(frameType !== 'SEI') {
                    // rtp时间戳周期为RESETTIME，如果单向递增，设为0
                    if(lastTimeStamp - rtpTimeStamp > (RESETTIME / 2)) { //判断lastTimeStamp远大于rtpTimeStamp，防止后一帧比前一帧先到的情况
                        //console.log(lastTimeStamp - rtpTimeStamp)
                        console.warn('时间戳重置', lastTimeStamp, rtpTimeStamp, frameType, new Date())
                        resetTimeCount ++;
                    }
                    rtpTimeStamp = rtpTimeStamp + RESETTIME * resetTimeCount;
                } else {
                    //同一帧的SEI比视频发送慢时
                    if (rtpTimeStamp - lastTimeStamp > (RESETTIME / 2)) {
                        console.warn('SEI翻转', rtpTimeStamp , lastTimeStamp);
                        rtpTimeStamp = rtpTimeStamp + RESETTIME * (resetTimeCount - 1);
                    } else {
                        rtpTimeStamp = rtpTimeStamp + RESETTIME * resetTimeCount;
                    }
                    //console.log('SEI', rtpTimeStamp, 'video', lastTimeStamp, lastTimeStamp+ RESETTIME * resetTimeCount - rtpTimeStamp)
                    // //同一帧的SEI比视频发送快时
                    // if(rtpTimeStamp - lastTimeStamp === 3600) {
                    //     if(rtpTimeStamp === 0 ) {
                    //         rtpTimeStamp = rtpTimeStamp + RESETTIME * (resetTimeCount + 1);
                    //     } else {
                    //         rtpTimeStamp = rtpTimeStamp + RESETTIME * resetTimeCount;
                    //     }
                    // }
                }

                if(frameType === 'SEI') {
                    //SEI被分片（nal === 28）时，分片包发送完后marker为1，不会和视频帧一起
                    SEIBuffer = inputBuffer.subarray(4, inputLength);
                    //console.log(SEIBuffer)
                    inputBufferSub = new Uint8Array();
                }
                if(SEIBuffer) {
                    let SEI = SEIParse(SEIBuffer);
                    if (SEI) {
                        SEIInfo.ivs = SEI;
                        SEIInfo.timestamp = rtpTimeStamp;
                        //console.log('GET SEI', rtpTimeStamp)
                        //SEI信息
                        decodedData.SEIInfo = SEIInfo;

                        if (!(rtpTimeStamp - lastTime)) {
                            console.log('上个SET包时间和本次相等');
                            console.log('lastTime: ', lastTime, ' rtpTimeStamp: ', rtpTimeStamp, RtpHeader.timeStamp, rtpHeader[4], rtpHeader[5], rtpHeader[6], rtpHeader[7])
                        }
                        if ((rtpTimeStamp - lastTime) !== 3600) {
                            //console.log('SEI 时间差：', (rtpTimeStamp - lastTime), rtpTimeStamp, lastTime)
                        }
                        lastTime = rtpTimeStamp;
                        SEIInfo = {
                            ivs: null,
                            timestamp: 0,
                        };
                    }
                    SEIBuffer = null;
                }

                if(decodeMode === 'canvas' && (frameType !== 'SEI')) {
                    if (outputSize !== curSize) {
                        outputSize = curSize;
                        decoder.setOutputSize(curSize);
                    }

                    decodedData.frameData = decoder.decode(inputBufferSub);
                    decodedData.decodeMode = 'canvas';
                    decodedData.timeStamp = rtpTimeStamp;
                    decodedData.width = width;
                    decodedData.height = height;
                    this.handleDecodedData(decodedData)
                } else {
                    if (!initalSegmentFlag) {
                        decodedData.initSegmentData = null;
                        decodedData.codecInfo = null;
                    } else {
                        initalSegmentFlag = false;
                        const info = {
                            id: 1,
                            width: width,
                            height: height,
                            type: "video",
                            profileIdc: SPSParser.getSpsValue("profile_idc"),
                            profileCompatibility: 0,
                            levelIdc: SPSParser.getSpsValue("level_idc"),
                            sps: [spsSegment],
                            pps: [ppsSegment],
                            timescale: 1e3,
                            fps: frameRate
                        };
                        decodedData.initSegmentData = info;
                        decodedData.codecInfo = SPSParser.getCodecInfo();
                        //console.log(info.pps)
                    }

                    if (frameType === 'I') {
//console.log('ppsSegment: ', ppsSegment)
                        let h264parameterLength = spsSegment.length + ppsSegment.length + 8;
                        inputSegBufferSub = inputBufferSub.subarray(h264parameterLength, inputBufferSub.length);
                    } else {
                        inputSegBufferSub = inputBufferSub.subarray(0, inputBufferSub.length);
                    }

                    let segSize = inputSegBufferSub.length - 4;
                    //mp4 box头
                    inputSegBufferSub[0] = (segSize & 0xFF000000) >>> 24;
                    inputSegBufferSub[1] = (segSize & 0xFF0000) >>> 16;
                    inputSegBufferSub[2] = (segSize & 0xFF00) >>> 8;
                    inputSegBufferSub[3] = (segSize & 0xFF);

                    decodedData.frameData = new Uint8Array(inputSegBufferSub);

                    let sample = {
                        duration: Math.round((1 / frameRate) * 1000),
                        size: inputSegBufferSub.length,
                        frame_time_stamp: null,
                        frameDuration: null,
                    };
                    sample.frame_time_stamp = rtpTimeStamp; //Todo：暂时为null，通过帧率控制duration
                    if (initalMediaFrameFlag) {
                        sample.frameDuration = 0;
                        initalMediaFrameFlag = false;
                    } else {
                        if(frameRate) {
                            sample.frameDuration = Math.round(1000 / frameRate);
                        }else {
                            sample.frameDuration = (sample.frame_time_stamp - preSample.frame_time_stamp) / 90; // 时钟频率90000，timescale=1000
                        }
                    }
                    preSample = sample;

                    decodedData.mediaSample = sample;

                    decodedData.timeStamp = rtpTimeStamp;

                    this.handleDecodedData(decodedData);
                }


                inputLength = 0;
                decodedData.SEIInfo = null;
                inputSegBufferSub = null;
                if(frameType !== 'SEI') {
                    lastTimeStamp = RtpHeader.timeStamp;
                }
                frameType = '';
            }
            //console.log('xxxxxxxxxxxxxxxxxxxxxxxxx')
        },

        set rtpSessionCallback(func) {
            this.handleDecodedData = func;
        },

        setFrameRate(fps) {
            frameRate = fps;
            //console.log('frameRate: ', frameRate)
        },

        setResolutionChangedCallback(callback) {
            this.resolutionChangedCallback = callback;
        }
    }

    return new constructor();

    function setBuffer(buffer1, buffer2) {
        let bufferTemp = buffer1;
        if ((inputLength + buffer2.length) > buffer1.length) {
            bufferTemp = new Uint8Array(buffer1.length + size1M);
        }

        bufferTemp.set(buffer2, inputLength);
        inputLength += buffer2.length;
        return bufferTemp;
    }
}




/**
 * 去除SPS中的Emulation字节
 * @param data SPS源数据
 * @returns {Array} 去除后Emulation字节后的SPS
 */
function removeH264or5EmulationBytes(data) {
    let toSize = 0;
    let i = 0;
    let to = [];
    let dataLength = data.length;
    while (i < dataLength) {
        if (i + 2 < dataLength && data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 3) {
            to[toSize] = to[toSize + 1] = 0;
            toSize += 2;
            i += 3;
        } else {
            to[toSize] = data[i];
            toSize += 1;
            i += 1;
        }
    }
    return to;
}

/**
 * 解析SEI信息
 * @param data
 * @return {Array}
 */
function SEIParse(data) {
    //console.log(data)
    if((data[0] & 0x1f) !== 6) {
        //非SEI
        return null;
    }

    if(data[1] !== 0x55 || data[2] !== 0x4C ||data[3] !== 0x53 ||data[4] !== 0x40) {
        //ULS@开头
        console.warn('unknown SEI type');
        return null;
    }

    let dataLength = data.length;
    let type = (data[5] << 8) + data[6];
    let checkSum = (data[7] << 8) + data[8];
    // if(dataLength !== (checkSum + 1 + 4 + 4)) {
    //     console.log('SEI check fail!');
    //     return null;
    // }
    let result;
    data = data.subarray(9);
    switch(type) {
        case 0:
            result = parseRect(data);
            break;
        case 1:
            //console.log(parseBody(data))
            result = parseBody(data);
            break;
        default:
            result = null;
            break;
    }
    return result;
}

function parseRect(data) {
    //console.log(data)
    let dataLength = data.length;
    let contents = [];
    while(dataLength > 0) {
        //console.log('dataLength: ', dataLength)
        let x0 = ((data[4] & 0x7f) << 8) + data[5],
            y0 = ((data[6] & 0x7f) << 8) + data[7],
            width = ((data[8] & 0x7f) << 8) + data[9] - x0,
            height = ((data[10] & 0x7f) << 8) + data[11] - y0;
        let content = {
            type: 'rect',
            id: (data[2] << 8) + data[3],
            rect: [x0, y0, width, height],
            state: data[1] & 0x01,
            quality: (data[1] & 0x02) >> 1,
        };
        Array.prototype.push.apply(contents, [content]);
        data = data.subarray(12);
        dataLength = data.length;
    }

    return contents;
}

function parseBody(data) {
    let dataLength = data.length;
    let contents = [];
    while(dataLength > 0) {
        //console.log('dataLength: ', dataLength)
        let x0 = ((data[4] & 0x7f) << 8) + data[5],
            y0 = ((data[6] & 0x7f) << 8) + data[7],
            width = ((data[8] & 0x7f) << 8) + data[9] - x0,
            height = ((data[10] & 0x7f) << 8) + data[11] - y0,
            boxConfidence = ((data[12] & 0x7f) << 8) + data[13];
        let points = [];

        for(let i = 0; i < 17; i++) {
            let point = {
                x: ((data[16 + i * 8] & 0x7f) << 8) + data[17 + i * 8],
                y: ((data[18 + i * 8] & 0x7f) << 8) + data[19 + i * 8],
                confidence: ((data[20 + i * 8] & 0x7f) << 8) + data[21 + i * 8],
                // x: Math.random() * 8191,
                // y: Math.random() * 8191,
                // confidence: 1,
            };
            points.push(point);
        }
        let content = {
            type: 'coco-pose',
            id: (data[2] << 8) + data[3],
            handsUp: data[1] & 0x04,
            boundingBox: [x0, y0, width, height],
            boxConfidence,
            points: parseBodyToTree(points),
            state: data[1] & 0x01,
        };
        Array.prototype.push.apply(contents, [content]);
        data = data.subarray(152);
        dataLength = data.length;

    }
    return contents;
}

/**
 * 将智能帧中的人体姿态点转化为树结构(双亲表示法)
 * @param points
 */
function parseBodyToTree(points) {
    let newPoints = [];
    newPoints[0] = {...points[0], parent: -1, pointColor:'#FF0002', lineColor: '#FF0002'};
    newPoints[1] = {...points[1], parent: 0, pointColor:'#FF0002', lineColor: '#FF0002'};
    newPoints[2] = {...points[2], parent: 0, pointColor:'#FF0002', lineColor: '#FF0002'};
    newPoints[3] = {...points[5], parent: 17, pointColor:'#D9E34F', lineColor: '#FF0002'};
    newPoints[4] = {...points[6], parent: 17, pointColor:'#D9E34F', lineColor: '#FF0002'};
    newPoints[5] = {...points[3], parent: 1, pointColor:'#00FF00', lineColor: '#FF0002'};
    newPoints[6] = {...points[4], parent: 2, pointColor:'#FFAC00', lineColor: '#FF0002'};
    newPoints[7] = {...points[7], parent: 3, pointColor:'#00FF45', lineColor: '#'};
    newPoints[8] = {...points[11], parent: 17, pointColor:'#EEC446', lineColor: '#'};
    newPoints[9] = {...points[8], parent: 4, pointColor:'#43D3AF', lineColor: '#'};
    newPoints[10] = {...points[12], parent: 17, pointColor:'#7A93E8', lineColor: '#'};
    newPoints[11] = {...points[9], parent: 7, pointColor:'#EFB842', lineColor: '#'};
    newPoints[12] = {...points[13], parent: 8, pointColor:'#E56C00', lineColor: '#'};
    newPoints[13] = {...points[10], parent: 9, pointColor:'#47CD43', lineColor: '#0096FF'};
    newPoints[14] = {...points[14], parent: 10, pointColor:'#3868D2', lineColor: '#00FF51'};
    newPoints[15] = {...points[15], parent: 12, pointColor:'#DF4D01', lineColor: '#5100FF'};
    newPoints[16] = {...points[16], parent: 14, pointColor:'#1E48D4', lineColor: '#00FFA0'};
    newPoints[17] = {x: (points[5].x + points[6].x) / 2,y: (points[5].y + points[6].y) / 2, parent: -1, pointColor:'#D9E34F', lineColor: '#00FFA0'};
    return newPoints;
}