workerManager.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523
  1. import VideoMediaSource from './MediaSource.js';
  2. import MP4Remux from './MP4Remux.js';
  3. import IvsDrawer from './ivsDrawer.js';
  4. function WorkerManager() {
  5. let videoWorker;
  6. let SDPInfo;
  7. let messageArray = [];
  8. let rtpStackCount = 0;
  9. let videoElement = null;
  10. let canvasElement = null;
  11. let videoMS = null;
  12. const rtpStackCheckNum = 10;
  13. let codecInfo = null;
  14. let initSegmentData = null;
  15. let mediaInfo = {
  16. id: 1,
  17. samples: null,
  18. baseMediaDecodeTime: 0
  19. };
  20. let numBox = 1;
  21. let mediaSegNum = 0; //用于记录缓存的box个数
  22. let mediaFrameData = null; //用于缓存未喂入mse的box
  23. let mediaFrameSize = 0; //mediaFrameData的大小
  24. let preBaseDecodeTime = 0; //上一个解码时间
  25. let curBaseDecodeTime = 0; //从第一帧到当前帧的持续时间
  26. let mediaSegmentData = null; //MP4化的数据
  27. let sequenseNum = 1;
  28. let mp4Remux;
  29. let firstTimeStamp = null; //第一个视频帧的时间戳
  30. let SEIinfo = null;
  31. let ivsDrawer = null;
  32. let info = null;
  33. let MAX_INFO = 25; // 限制info最大长度
  34. let startDrawIVS = false;
  35. let onCanplayCallback = null;
  36. let ROIdrawer = null;
  37. let initialCompleted = false;
  38. let onInitialCompletedCallback = null;
  39. let frameRate = 25; //默认25
  40. let resizeObserver = null;
  41. function constructor() {
  42. }
  43. constructor.prototype = {
  44. init(video,canvas,ROIElement) {
  45. videoWorker = new Worker('./src/videoWorker.js');
  46. videoWorker.onmessage = videoWorkerMessage;
  47. videoElement = video;
  48. canvasElement = canvas;
  49. mp4Remux = new MP4Remux();
  50. mp4Remux.init();
  51. SEIinfo = new IVSQueue();
  52. info = new LruCache(MAX_INFO);
  53. ivsDrawer = new IvsDrawer(canvasElement);
  54. ROIdrawer = ROIElement;
  55. resizeObserver = new ResizeObserver( entries => {
  56. ivsDrawer.cover(videoElement);
  57. ROIdrawer.cover(videoElement);
  58. ROIdrawer.redrawROI();
  59. });
  60. resizeObserver.observe(videoElement);
  61. },
  62. sendSdpInfo(SDPinfo) {
  63. SDPInfo = SDPinfo;
  64. console.log(SDPinfo)
  65. let message = {
  66. type: "sdpInfo",
  67. data: {
  68. sdpInfo: SDPInfo
  69. }
  70. };
  71. videoWorker.postMessage(message);
  72. //Todo:暂时采用第一路视频的帧率
  73. if(SDPInfo.length && SDPInfo[0].Framerate) {
  74. frameRate = SDPinfo[0].Framerate;
  75. }
  76. },
  77. parseRtpData(rtspinterleave, rtpheader, rtpPacketArray) {
  78. // console.log(rtspinterleave)
  79. // console.log( rtpheader)
  80. // //console.log(rtpPacketArray)
  81. // console.log(rtpheader[3])
  82. let mediaType = rtspinterleave[1];
  83. let idx = parseInt(mediaType / 2, 10);
  84. let markerBitHex = 128;
  85. let message = {
  86. type: "rtpData",
  87. data: {rtspInterleave: rtspinterleave, header: rtpheader, payload: rtpPacketArray}
  88. };
  89. //console.log(rtspinterleave)
  90. //console.log('idx: ',idx)
  91. if(idx !== 0) {
  92. console.log('idx: ',rtspinterleave);
  93. //console.log(SDPInfo)
  94. return;
  95. }
  96. switch (SDPInfo[idx].codecName) {
  97. case"H264":
  98. messageArray.push(message);
  99. if (rtpStackCount >= rtpStackCheckNum || (rtpheader[1] & markerBitHex) === markerBitHex) {
  100. if((rtpheader[1] & markerBitHex) === markerBitHex) {
  101. //onsole.log('遇到终止位: ' + rtpheader[1])
  102. }
  103. let sendMessage = {type: "rtpDataArray", data: messageArray};
  104. if (videoWorker) {
  105. videoWorker.postMessage(sendMessage)
  106. }
  107. sendMessage = null;
  108. messageArray = [];
  109. rtpStackCount = 0
  110. //console.log('1111111111')
  111. } else {
  112. rtpStackCount++
  113. }
  114. break;
  115. default:
  116. }
  117. },
  118. /**
  119. * 更新需要绘制的其它信息
  120. * @param obj
  121. */
  122. updateInfo(obj) {
  123. //if((obj.name !== '') && (obj.name !== undefined) && (obj.name !== null)) {
  124. info.set(obj.id, obj.name);
  125. //}
  126. },
  127. setEventCallBack(event, callback) {
  128. switch (event) {
  129. case 'canplay':
  130. onCanplayCallback = callback;
  131. break;
  132. case 'initialCompleted':
  133. onInitialCompletedCallback = callback;
  134. break;
  135. case 'ROIFinished':
  136. ROIdrawer.setROIFinishedCallback(callback);
  137. break;
  138. default:
  139. break;
  140. }
  141. },
  142. setROIDrawer(ROIElement) {
  143. ROIdrawer = ROIElement;
  144. },
  145. terminate() {
  146. videoWorker.terminate();
  147. ivsDrawer.terminate();
  148. resizeObserver && resizeObserver.unobserve(videoElement);
  149. resizeObserver = null;
  150. info.clear();
  151. startDrawIVS = false;
  152. window.onresize = null;
  153. if(videoMS) {
  154. videoMS.close();
  155. videoMS = null;
  156. }
  157. }
  158. }
  159. return new constructor();
  160. function videoWorkerMessage(event) {
  161. let videoMessage = event.data;
  162. let type = videoMessage.type;
  163. //console.log(videoMessage.data)
  164. switch (type) {
  165. // case 'codecInfo': //设置codecType
  166. // break;
  167. // case 'initSegment': //第一个buffer,设置SPS等
  168. case 'videoInit'://合并codecInfo和initSegment
  169. console.log(videoMessage)
  170. codecInfo = videoMessage.data.codecInfo;
  171. //console.log(videoMessage.data)
  172. initSegmentData = mp4Remux.initSegment(videoMessage.data.initSegmentData);
  173. //console.log(initSegmentData)
  174. videoMS = new VideoMediaSource(videoElement);
  175. videoMS.CodecInfo = codecInfo;
  176. videoMS.InitSegment = initSegmentData;
  177. //console.log(videoMS.CodecInfo, videoMS.InitSegment)
  178. videoMS.init();
  179. videoMS.onCanplayCallback(()=>{
  180. ivsDrawer.cover(videoElement);
  181. onCanplayCallback && onCanplayCallback();
  182. if(!initialCompleted) {
  183. ROIdrawer.cover(videoElement);
  184. onInitialCompletedCallback && onInitialCompletedCallback();
  185. initialCompleted = true;
  186. }
  187. });
  188. break;
  189. case 'firstvideoTimeStamp':
  190. firstTimeStamp = videoMessage.data;
  191. videoMS.setFirstTimeStamp(firstTimeStamp);
  192. //videoMS.setDurationChangeCallBack(drawIVS);
  193. console.log('first frame timestamp: ', firstTimeStamp);
  194. startDrawIVS = true;
  195. window.requestAnimationFrame(draw);
  196. break;
  197. case 'videoTimeStamp'://时间戳,用于智能同步
  198. //videoMS.setFirstTimeStamp(videoMessage.data);
  199. //console.log('frame timestamp: ', videoMessage.data);
  200. //console.log('npt: ', ( videoMessage.data - firstTimeStamp)/90000)
  201. break;
  202. case 'mediaSample': //用于设置baseMediaDecodeTime
  203. if(mediaInfo.samples == null) {
  204. mediaInfo.samples = new Array(numBox);
  205. }
  206. //console.log('frameDuration: ' + videoMessage.data.frameDuration)
  207. curBaseDecodeTime += videoMessage.data.frameDuration;
  208. mediaInfo.samples[mediaSegNum++] = videoMessage.data;
  209. break;
  210. case 'videoRender': //视频数据
  211. //缓存该segment数据
  212. let tempBuffer = new Uint8Array(videoMessage.data.length + mediaFrameSize);
  213. if(mediaFrameSize !== 0) {
  214. tempBuffer.set(mediaFrameData);
  215. }
  216. //console.log(videoMessage)
  217. tempBuffer.set(videoMessage.data, mediaFrameSize);
  218. mediaFrameData = tempBuffer;
  219. mediaFrameSize = mediaFrameData.length;
  220. if(mediaSegNum % numBox === 0 && mediaSegNum !== 0) {
  221. if (sequenseNum === 1) {
  222. mediaInfo.baseMediaDecodeTime = 0
  223. } else {
  224. mediaInfo.baseMediaDecodeTime = preBaseDecodeTime;
  225. }
  226. preBaseDecodeTime = curBaseDecodeTime;
  227. mediaSegmentData = mp4Remux.mediaSegment(sequenseNum, mediaInfo, mediaFrameData);
  228. sequenseNum++;
  229. mediaSegNum = 0;
  230. mediaFrameData = null;
  231. mediaFrameSize = 0;
  232. if (videoMS !== null) {
  233. videoMS.setMediaSegment(mediaSegmentData)
  234. } else {
  235. }
  236. }
  237. break;
  238. case 'YUVData'://FFMPEG解码的数据
  239. //console.log(videoMessage.data)
  240. //draw(videoMessage.data);
  241. //yuv2canvas(videoMessage.data.data, videoMessage.data.width, videoMessage.data.height,canvasElement)
  242. break;
  243. case 'SEI': //处理SEI信息
  244. //console.log('SEI timestamp: ', videoMessage.data.timestamp);
  245. //console.log('SEI-npt: ', (videoMessage.data.timestamp - firstTimeStamp)/90000)
  246. if(videoMessage.data.ivs !== null) {
  247. let ivs = [];
  248. videoMessage.data.ivs.map((content, k) => {
  249. if(content.state) { //state=1, 绘制该信息
  250. ivs.push(content);
  251. }else { //state=0, 清除info中对应的id:name
  252. // let id = content.id;
  253. // console.log('删除', id, info[id]);
  254. // delete info[id];
  255. // console.log(info)
  256. }
  257. });
  258. //console.log('PUSH SEI: ', videoMessage.data.timestamp)
  259. SEIinfo.push(videoMessage.data.timestamp, ivs);
  260. //console.log(videoMessage.data.timestamp - lastTime)
  261. //lastTime = videoMessage.data.timestamp;
  262. }
  263. //console.log('timestamp: ', videoMessage.data.timestamp)
  264. //console.log(SEIinfo)
  265. break;
  266. default:
  267. console.log('暂不支持其他类型');
  268. break;
  269. }
  270. }
  271. function draw() {
  272. let timestamp = parseInt((videoElement.currentTime.toFixed(2) * 90000).toFixed(0)) + firstTimeStamp + 90000/frameRate;//
  273. drawIVS(timestamp);
  274. if(startDrawIVS) {
  275. window.requestAnimationFrame(draw);
  276. }
  277. }
  278. /**
  279. * 根据时间戳获取相应的ivs信息
  280. * @param timestamp 当前帧的时间戳
  281. * @returns {*} ivs信息
  282. */
  283. function getIVS(timestamp) {
  284. let preNode = null;
  285. let nextNode = null;
  286. preNode = SEIinfo.shift();
  287. nextNode = SEIinfo.top();
  288. while((preNode !== undefined) && (preNode !== null)) {
  289. if(preNode[0] > timestamp) {
  290. SEIinfo.unshift(preNode);
  291. //console.log('SEI时间大于video: ', preNode[0], timestamp);
  292. return null;
  293. } else if(preNode[0] === timestamp) {
  294. return preNode;
  295. } else {
  296. if(nextNode === undefined || nextNode === null) {
  297. console.log('last ivs info: ', timestamp, preNode[0], SEIinfo);
  298. //console.log(preNode[0] - lastTime);
  299. //lastTime = preNode[0];
  300. if(timestamp - preNode[0] < 90000/frameRate) {
  301. return preNode;//最后一个node
  302. }
  303. return null;
  304. }
  305. if(nextNode[0] > timestamp) {
  306. // console.log('video time: ', timestamp, preNode[0], SEIinfo.length());
  307. // if(SEIinfo.length()) {
  308. // SEIinfo.map((v, k)=>{
  309. // console.log(v);
  310. // });
  311. // }
  312. //console.log(preNode[0] - lastTime);
  313. //lastTime = preNode[0];
  314. return preNode;
  315. } else if(nextNode[0] === timestamp){
  316. nextNode = SEIinfo.shift();
  317. //console.log('video time: ', timestamp, nextNode[0], SEIinfo);
  318. //console.log(nextNode[0] - lastTime);
  319. //lastTime = nextNode[0];
  320. return nextNode;
  321. } else {
  322. preNode = SEIinfo.shift();
  323. nextNode = SEIinfo.top();
  324. }
  325. }
  326. }
  327. return null;
  328. }
  329. /**
  330. * 绘制智能信息
  331. * @param timestamp
  332. */
  333. function drawIVS(timestamp) {
  334. let data = getIVS(timestamp);
  335. if(data === undefined || data === null) {
  336. //清空画布
  337. //ivsDrawer.clearCanvas();
  338. if(!SEIinfo.length() || (SEIinfo.length() && (SEIinfo.list[0][0] - timestamp) > 90000/frameRate)) {
  339. ivsDrawer.clearCanvas();
  340. }
  341. }else {
  342. //console.log('GET SEI: ', data[0], ' videoTimestamp', timestamp);
  343. data = data[1];
  344. //console.log(info.map.length)
  345. if(info.map.length > MAX_INFO) {
  346. console.log('info length: ', info.map.length);
  347. }
  348. //获取鹰眼信息
  349. data.map((content, k) =>{
  350. let result = info.get(content.id);
  351. if(result !== undefined && result !== null) {
  352. data[k].text = result.value;
  353. }
  354. });
  355. ivsDrawer.draw(data, timestamp);
  356. }
  357. }
  358. }
  359. function ElementResizeCallback(someElement, callback) {
  360. const ro = new ResizeObserver( entries => {
  361. // for (let entry of entries) {
  362. // const cr = entry.contentRect;
  363. // console.log('Element:', entry.target);
  364. // console.log(`Element size: ${cr.width}px x ${cr.height}px`);
  365. // }
  366. callback();
  367. });
  368. ro.observe(someElement);
  369. }
  370. function yuv2canvas(yuv, width, height, canvas) {
  371. canvas.width = width;
  372. canvas.height = height;
  373. var context = canvas.getContext("2d");
  374. var output = context.createImageData(width, height);
  375. var outputData = output.data;
  376. var yOffset = 0;
  377. var uOffset = width * height;
  378. var vOffset = width * height + (width*height)/4;
  379. for (var h=0; h<height; h++) {
  380. for (var w=0; w<width; w++) {
  381. var ypos = w + h * width + yOffset;
  382. var upos = (w>>1) + (h>>1) * width/2 + uOffset;
  383. var vpos = (w>>1) + (h>>1) * width/2 + vOffset;
  384. var Y = yuv[ypos];
  385. var U = yuv[upos] - 128;
  386. var V = yuv[vpos] - 128;
  387. var R = (Y + 1.371*V);
  388. var G = (Y - 0.698*V - 0.336*U);
  389. var B = (Y + 1.732*U);
  390. var outputData_pos = w*4 + width*h*4;
  391. outputData[0+outputData_pos] = R;
  392. outputData[1+outputData_pos] = G;
  393. outputData[2+outputData_pos] = B;
  394. outputData[3+outputData_pos] = 255;
  395. }
  396. }
  397. context.putImageData(output, 0, 0);
  398. }
  399. class IVSQueue {
  400. constructor() {
  401. this.list = [];
  402. }
  403. push(timestamp, ivs) {
  404. for (let i = 0, len = this.list.length; i < len; i++) {
  405. if(this.list[i][0] === timestamp) {
  406. let oldIvs = this.list[i][1];
  407. Array.prototype.push.apply(oldIvs, ivs);
  408. this.list[i][1] = oldIvs;
  409. return;
  410. }
  411. }
  412. this.list.push([timestamp, ivs]);
  413. }
  414. shift() {
  415. let tmp = this.list.shift();
  416. return tmp;
  417. }
  418. unshift(node) {
  419. this.list.unshift(node);
  420. }
  421. top() {
  422. let tmp = this.list[0];
  423. return tmp;
  424. }
  425. length() {
  426. return this.list.length;
  427. }
  428. map(v,k) {
  429. return this.list.map(v,k);
  430. }
  431. }
  432. class LruCache {
  433. constructor(limit) {
  434. this.limit = limit || 20;
  435. this.map = [];
  436. }
  437. get(key) {
  438. return this._search(key);
  439. }
  440. set(key, value) {
  441. let result = this._search(key);
  442. if(!result) {
  443. this.map.unshift({
  444. key: key,
  445. value: value
  446. });
  447. if(this.map.length > this.limit) {
  448. this.map.pop();
  449. }
  450. }
  451. }
  452. //每次查找将该元素置于队首
  453. _search(key) {
  454. for(let i = 0, length = this.map.length; i < length; i++) {
  455. if(this.map[i].key === key) {
  456. let head = this.map.splice(i, 1);
  457. this.map.unshift(head[0]);
  458. return head[0];
  459. }
  460. }
  461. return null;
  462. }
  463. clear() {
  464. this.map = [];
  465. }
  466. }
  467. export default WorkerManager;