DengQN·一个普通程序员;
Crudboy日常】对Ceisum视频融合的探索
2024-05-09 18:17 171
#cesium#视频

目标

把视频流显示到cesium场景里，和场景模型混合，达到视频融合的效果
类似

基本思路是生成一个摄像机，然后把他对应的frustum取出来，把视频流作为材质贴到它的远截面。

摄像头协议过来的数据会带有摄像头的姿态参数，直接用来控制这个摄像头实例。

QQ截图20240509181855.png

创建项目

创建一个vue项目

npm install cesium

画出四棱锥？

应该是叫四棱台吧。。四棱锥？

先是创建摄像头实例

camera.value = new Camera(viewer.scene);
    camera.value.position = cameraPosition.value;
    camera.value.direction = Cartesian3.negate(Cartesian3.UNIT_Z, new Cartesian3());
    camera.value.up = Cartesian3.clone(Cartesian3.UNIT_Y);
    camera.value.frustum.fov = Math.PI_OVER_THREE;

直接new就完事了， 其中，position 是到时候场景中摄像头的实际位置， 其他的给些默认参数就行了。

有摄像头了，可以直接取出它的frustum来创建图元。

先给个默认

const frustum = ref(new PerspectiveFrustum({
    fov: fov.value,
    aspectRatio: aspectRatio.value,
    near: near.value,
    far: far.value
}))

fov, ratio, near， far 这些参数都给个默认值，到时候正式数据过来直接改了，创建新的就行。

类型定义：
参考链接 -> PerspectiveFrustum

其实我们创建的是它的边框

frustumOutLineGeo.value = new FrustumOutlineGeometry({
        frustum: frustum.value,
        origin: cameraPosition.value,
        orientation: Quaternion.fromHeadingPitchRoll(new HeadingPitchRoll(heading.value, pitch.value, roll.value)),
    })

然后是实例

frustumOutLineGeoInst.value = new GeometryInstance({
      geometry: frustumOutLineGeo.value,
      attributes: {
          color: ColorGeometryInstanceAttribute.fromColor(
              Color.YELLOWGREEN
          ),
          show: new ShowGeometryInstanceAttribute(true)
      }
  })

其中！attributes的内容，很重要，不写或写错会报错，然后不好排查。。

最后是新建图元

let frustumOutLinePrimitive = new Primitive({
        geometryInstances: [frustumOutLineGeoInst.value],
        appearance: new PerInstanceColorAppearance({
            closed: true,
            flat: true
        }),
        asynchronous: false,
    })

塞到场景里就好了

viewer.scene.primitives.add(frustumOutLinePrimitive)

然后就可以得到一个四棱台在场景里了

获取远截面坐标点

四棱台的远截面坐标点可以通过四棱台的图元和geometryInstance计算得到

此处拼拼凑凑一些dalao的代码得到

/ 获取视锥体远端面顶点坐标
function getCorners(primitive, geometryInstance, drawPoint = false) {
    // 获取 primitive 的几何体实例
    // const instanceGeo = primitive.geometryInstances[0]

    // 获取几何体实例的几何体对象, 关键！
    const geometry = FrustumOutlineGeometry.createGeometry(geometryInstance.geometry)
    // console.log("primitive: ", primitive);
    // console.log("geometry: ", geometry);
    // 获取几何体的顶点属性
    const attributes = geometry.attributes

    // 获取顶点属性中的位置属性
    const positions = attributes.position.values

    // 获取图元转置矩阵
    const modelMatrix = primitive.modelMatrix

    // 清空顶点几何体
    // this.clearSphere()

    const resultArr = []
    // 前4个顶点刚好为远端面顶点
    for (let i = 3 * 4; resultArr.length < 4; i += 3) {
        // 创建顶点坐标的 Cartesian4 对象
        const x = positions[i]
        const y = positions[i + 1]
        const z = positions[i + 2]
        const vertex = new Cartesian4(x, y, z, 1.0)
        // 应用模型矩阵到顶点坐标
        Matrix4.multiplyByPoint(modelMatrix, vertex, vertex)
        // 将顶点坐标转换为地理坐标
        const cartesian3 = Cartesian3.fromCartesian4(vertex)
        // const cartographic = Cartographic.fromCartesian(cartesian3)
        resultArr.push(cartesian3)
    }
    let ellsd = viewer.scene.globe.ellipsoid;


    let cartesian3 = resultArr //.map(cartographic => ellsd.cartographicToCartesian(cartographic))

    if (drawPoint) {
        for (var i = 0; i < cartesian3.length; i++) {
            viewer.entities.removeById('far-clip-' + i)
            viewer.entities.add({
                id: 'far-clip-' + i,
                position: cartesian3[i],
                point: {
                    pixelSize: 5,
                    color: Color.RED,
                    outlineColor: Color.WHITE,
                    outlineWidth: 2,
                },
                label: {
                    text: 'point' + (i + 1),
                }
            })
        }
    }

    return cartesian3;
}

那么，得到了四个坐标点。

视频多边形绘制

直接创建一个多边形图元

// 视频
  // m.uniforms.image = videoElement.value;
  const primitive = new Primitive({
      asynchronous: false,
      geometryInstances: new GeometryInstance({
          geometry: new PolygonGeometry({

              perPositionHeight: true,
              polygonHierarchy: new PolygonHierarchy(controlPoints),
              // vertexFormat: PolylineColorAppearance.VERTEX_FORMAT
          })
      }),
      appearance: new MaterialAppearance({
          material: material,
          renderState: {
              blending: BlendingState.ALPHA_BLEND
          }
      })
  })
  videoPrimitive.value = viewer.scene.primitives.add(primitive)

关键点在于 appearance 参数使用MaterialAppearance， 并将我们自定义的材质传递进去

videoElement.value = document.getElementById('v');
var material = Material.fromType('Image');
material.uniforms.image = videoElement.value;

v就是个video标签，加载视频流的时候用video.js之类的就行了，这里我直接填MP4 url了。

参数控制

function listenAll(callback, ...values) {
    for (var value of values) {
        watch(value, callback)
    }
}

listenAll(function () {
  // 更新Frustum， new个新的
    changeFrustum()
    addFrustum(function () {
        let controlPoints = getCorners(frustumOutLinePrimitiveI.value, frustumOutLineGeoInst.value, true)
        console.log("corners: ", controlPoints);
        addPrimitiveVideo(controlPoints, wallAlpha.value)
    })
}, fov, heading, pitch, roll, aspectRatio, near, far)

当相机参数改变的时候，自动更新相关图元。

全部代码

<template>
    <div style="display: flex; flex-direction: row;  justify-content: center; align-items: center; min-height: 1000px;">
        <div id="cesiumContainer" style="width: 60%; min-width: 800px; padding-top: 100px;"></div>
        <div style="display: flex; flex-direction: row; flex-wrap: wrap; width: 40%;">
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">wallAlpha</span> <n-slider
                    disabled="" style="margin-top: 10px;" v-model:value="wallAlpha" :step="0.01" :max="1.0" :min="0.0"
                    w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">heading [{{ heading }}]</span>
                <n-slider style="margin-top: 10px;" v-model:value="heading" :step="0.01" :max="Math.PI-0.01" :min="-Math.PI+0.01"
                    w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">pitch [{{ pitch }}]</span> <n-slider
                    style="margin-top: 10px;" v-model:value="pitch" :step="0.01" :max="Math.PI-0.01" :min="-Math.PI+0.01"
                    w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">roll [{{ roll }}]</span> <n-slider
                    style="margin-top: 10px;" v-model:value="roll" :step="0.01" :max="Math.PI-0.01" :min="-Math.PI+0.01"
                    w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">fov [{{ fov }}]</span> <n-slider
                    style="margin-top: 10px;" v-model:value="fov" :step="0.01" :max="1" :min="0.01" w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">aspectRatio [{{ aspectRatio }}]</span>
                <n-slider style="margin-top: 10px;" v-model:value="aspectRatio" :step="0.01" :max="2" :min="0.01"
                    w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">far [{{ far }}]</span> <n-slider
                    style="margin-top: 10px;" v-model:value="far" :step="1" :max="1000" :min="near + 1" w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">near [{{ near }}]</span> <n-slider
                    style="margin-top: 10px;" v-model:value="near" :step="1" :max="far - 1" :min="1" w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">offsetX [{{ offsetX }}]</span>
                <n-slider style="margin-top: 10px;" v-model:value="offsetX" :step="1" :max="100" :min="-100"
                    w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">offsetY [{{ offsetY }}]</span>
                <n-slider style="margin-top: 10px;" v-model:value="offsetY" :step="1" :max="100" :min="-100"
                    w></n-slider></div>
            <div style="width: 100%; padding-left: 2%;"> <span style="width: 30px">offsetZ [{{ offsetZ }}]</span>
                <n-slider style="margin-top: 10px;" v-model:value="offsetZ" :step="1" :max="100" :min="-100"
                    w></n-slider></div>

            <div style="width: 100%; padding-left: 2%;"><n-button @click="toCamera">到摄像头位置</n-button></div>
            <div style="width: 100%; padding-left: 2%;"><n-button @click="resetCamera">到摄像头位置</n-button></div>
            
        </div>
        <video crossorigin="anonymous" style="width: 100px; height: 100px; rotate: 90; position: fixed; top: 0; opacity: 0;" id="v" loop="" class="video-js" autoplay="autoplay" preload="auto" muted>
        <!-- <video crossorigin="anonymous" style="width: 100px; height: 100px; position: fixed; top: 0; " id="v" loop="" class="video-js" autoplay="autoplay" preload="auto" muted> -->
            <!-- <source :src="'/api/file/video/8f4ab78c-361f-753f-93b4-355940f1e497.MP4'"> -->
            <!-- <source :src="'/api/file/video/1707909659202.mov'"> -->
            <source :src="'/api/file/video/8f4ab78c-361f-753f-93b4-355940f1e497.MP4'">
        </video>
    </div>
</template>
<script setup>

import { NSlider, NButton } from 'naive-ui';

import {
    Ion,
    Viewer,
    Cartesian3,
    PerspectiveFrustum,
    FrustumGeometry,
    Quaternion,
    VertexFormat,
    GeometryInstance,
    Color,
    Camera,
    Primitive,
    PerInstanceColorAppearance,
    FrustumOutlineGeometry,
    HeadingPitchRoll,
    ShowGeometryInstanceAttribute,
    ColorGeometryInstanceAttribute,
    CallbackProperty,
    Matrix4,
    Cartesian4,
    Cartographic,
    GroundPrimitive,
    PolygonGeometry,
    PolygonHierarchy,
    PolylineColorAppearance,
    BlendingState,
    MaterialAppearance,
    Material,
    PolylineMaterialAppearance,
    WallGeometry
} from 'cesium';
import { onMounted, ref, watch } from 'vue';

/**
 * controls
 */
const wallAlpha = ref(0.65)

const heading = ref(0)
const pitch = ref(0)
const roll = ref(0)
const fov = ref(90 / 360)
const aspectRatio = ref(1)
const near = ref(10)
const far = ref(50)

const offsetX = ref(0)
const offsetY = ref(0)
const offsetZ = ref(0)


const camera = ref()
const originCamera = ref()

var viewer = null;

const cameraPosition = ref(Cartesian3.fromDegrees(108.365386, 22.817802, 50))

const frustum = ref(new PerspectiveFrustum({
    fov: fov.value,
    aspectRatio: aspectRatio.value,
    near: near.value,
    far: far.value
}))

const frustumOutLineGeo = ref(null)
const frustumOutLineGeoInst = ref(null)
const frustumOutLinePrimitiveI = ref(null)

const videoElement = ref(null)
const videoPrimitive = ref(null)
// const cameraPrimitive = ref(null)

listenAll(function () {
    // console.log("frustumOutLinePrimitiveI", frustumOutLinePrimitiveI);
    // frustumOutLinePrimitiveI.value.update()
    changeFrustum()
    addFrustum(function () {
        let controlPoints = getCorners(frustumOutLinePrimitiveI.value, frustumOutLineGeoInst.value, true)
        console.log("corners: ", controlPoints);
        addPrimitiveVideo(controlPoints, wallAlpha.value)
    })
}, fov, heading, pitch, roll, aspectRatio, near, far)

// watch(fov, function () { 
//     console.log("frustumOutLinePrimitiveI", frustumOutLinePrimitiveI);
//     // frustumOutLinePrimitiveI.value.update()
//     changeFrustum()
//     addFrustum()
//  })

function listenAll(callback, ...values) {
    for (var value of values) {
        watch(value, callback)
    }
}

function changeFrustum() {
    frustum.value = new PerspectiveFrustum({
        fov: fov.value,
        aspectRatio: aspectRatio.value,
        near: near.value,
        far: far.value
    })
}

onMounted(async () => {

    Ion.defaultAccessToken = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI4MGRiNDk3ZS1kODE4LTQwMTUtYTYyZi00YTRjMDcwOGExMTQiLCJpZCI6MTI3OTQ4LCJpYXQiOjE2NzgzNDU0Nzl9.3-Pe0ofpxcuklDCi9XECUjbskAKfMZuZRJFuLPXrltg';

    viewer = new Viewer('cesiumContainer', {
        geocoder: false,
        homeButton: false,
        sceneModePicker: false,
        baseLayer: false,
        navigationHelpButton: false,
        animation: false,
        timeline: false,
        fullscreenButton: false,

        shouldAnimate: true,
        infoBox: false,
        vrButton: false,
        // terrainProvider:  createWorldTerrain()
        // terrainProvider: createWorldTerrain()
    })
    viewer._cesiumWidget._creditContainer.style.display = "none";
    viewer.scene.skyAtmosphere.show = true
    viewer.scene.globe.depthTestAgainstTerrain = true
    // viewer.scene.globe.show = false
    // 3. load video stream
    videoElement.value = document.getElementById('v');



    camera.value = new Camera(viewer.scene);
    camera.value.position = cameraPosition.value;
    camera.value.direction = Cartesian3.negate(Cartesian3.UNIT_Z, new Cartesian3());
    camera.value.up = Cartesian3.clone(Cartesian3.UNIT_Y);
    camera.value.frustum.fov = Math.PI_OVER_THREE;
    // camera.value.frustum.near = 1.0;
    // camera.value.frustum.far = 2.0;
    originCamera.value = viewer.camera

    addFrustum(function () {
        let controlPoints = getCorners(frustumOutLinePrimitiveI.value, frustumOutLineGeoInst.value, true)
        console.log("corners: ", controlPoints);
        addPrimitiveVideo(controlPoints, wallAlpha.value)
    })
    // console.log("frumstum: ", cameraPrimitive.value);
    viewer.camera.flyTo({ destination: cameraPosition.value })


})


function toCamera() {
    viewer.camera = camera.value
}

function resetCamera() {
    viewer.camera = originCamera.value
}


function addFrustum(callback) {
    if (frustumOutLinePrimitiveI.value) {
        console.log(viewer.scene.primitives);
        viewer.scene.primitives.removeAll()
        console.log(viewer.scene.primitives);
    }

    // 1
    // let frustum = camera.value.frustum
    // let frustum = ;
    // console.log("camera: ", camera.value);
    // 2
    let frustumGeo = new FrustumGeometry({
        frustum: frustum.value,
        origin: cameraPosition.value,
        orientation: Quaternion.fromHeadingPitchRoll(new HeadingPitchRoll(heading.value, pitch.value, roll.value)),
        vertexFormat: VertexFormat.POSITION_ONLY
    });

    frustumOutLineGeo.value = new FrustumOutlineGeometry({
        frustum: frustum.value,
        origin: cameraPosition.value,
        orientation: Quaternion.fromHeadingPitchRoll(new HeadingPitchRoll(heading.value, pitch.value, roll.value)),
    })

    // // 3
    let frustumGeoInst = new GeometryInstance({
        geometry: frustumGeo,
        attributes: {
            color: ColorGeometryInstanceAttribute.fromColor(
                Color.YELLOWGREEN
            ),
            show: new ShowGeometryInstanceAttribute(true)
        }
    });

    frustumOutLineGeoInst.value = new GeometryInstance({
        geometry: frustumOutLineGeo.value,
        attributes: {
            color: ColorGeometryInstanceAttribute.fromColor(
                Color.YELLOWGREEN
            ),
            show: new ShowGeometryInstanceAttribute(true)
        }
    })
    // // 4
    let frustumPrimitive = new Primitive({
        geometryInstances: [frustumGeoInst],
        appearance: new PerInstanceColorAppearance({
            closed: true,
            flat: true,

        }),

        asynchronous: false
    })

    let frustumOutLinePrimitive = new Primitive({
        geometryInstances: [frustumOutLineGeoInst.value],
        appearance: new PerInstanceColorAppearance({
            closed: true,
            flat: true
        }),
        asynchronous: false,
    })

    // let frustumPrimitiveI = viewer.scene.primitives.add(frustumPrimitive)
    frustumOutLinePrimitiveI.value = viewer.scene.primitives.add(frustumOutLinePrimitive)

    callback()
}

// 获取视锥体远端面顶点坐标
function getCorners(primitive, geometryInstance, drawPoint = false) {
    // 获取 primitive 的几何体实例
    // const instanceGeo = primitive.geometryInstances[0]

    // 获取几何体实例的几何体对象, 关键！
    const geometry = FrustumOutlineGeometry.createGeometry(geometryInstance.geometry)
    // console.log("primitive: ", primitive);
    // console.log("geometry: ", geometry);
    // 获取几何体的顶点属性
    const attributes = geometry.attributes

    // 获取顶点属性中的位置属性
    const positions = attributes.position.values

    // 获取图元转置矩阵
    const modelMatrix = primitive.modelMatrix

    // 清空顶点几何体
    // this.clearSphere()

    const resultArr = []
    // 前4个顶点刚好为远端面顶点
    for (let i = 3 * 4; resultArr.length < 4; i += 3) {
        // 创建顶点坐标的 Cartesian4 对象
        const x = positions[i]
        const y = positions[i + 1]
        const z = positions[i + 2]
        const vertex = new Cartesian4(x, y, z, 1.0)
        // 应用模型矩阵到顶点坐标
        Matrix4.multiplyByPoint(modelMatrix, vertex, vertex)
        // 将顶点坐标转换为地理坐标
        const cartesian3 = Cartesian3.fromCartesian4(vertex)
        // const cartographic = Cartographic.fromCartesian(cartesian3)
        resultArr.push(cartesian3)
    }
    let ellsd = viewer.scene.globe.ellipsoid;


    let cartesian3 = resultArr //.map(cartographic => ellsd.cartographicToCartesian(cartographic))

    if (drawPoint) {
        for (var i = 0; i < cartesian3.length; i++) {
            viewer.entities.removeById('far-clip-' + i)
            viewer.entities.add({
                id: 'far-clip-' + i,
                position: cartesian3[i],
                point: {
                    pixelSize: 5,
                    color: Color.RED,
                    outlineColor: Color.WHITE,
                    outlineWidth: 2,
                },
                label: {
                    text: 'point' + (i + 1),
                }
            })
        }
    }

    return cartesian3;
}

// 贴地
async function addGroundPrimitiveVideo(controlPoints, alpha) {


    if (videoPrimitive.value) {
        viewer.scene.primitives.remove(videoPrimitive.value)
    }


    let shaderSource = `
        uniform float alpha;
        // uniform vec2 repeat_1;

        czm_material czm_getMaterial(czm_materialInput materialInput)
            {
                vec4 color = texture2D(image, materialInput.st);
                czm_material material = czm_getDefaultMaterial(materialInput);
                // material.diffuse = czm_gammaCorrect(texture(image_0, fract(vec2(1) * materialInput.st)).rgb * vec4(1).rgb);
                material.diffuse = color.rgb;
                material.alpha = alpha;

                //if ((materialInput.st.s > 0.5 && materialInput.st.s < 0.6) && (materialInput.st.t > 0.5 && materialInput.st.t < 0.8)) {
                //   material.diffuse = vec3(0.3, 1, 0.2);
                //


                // vec2 center = vec2(0.5, 0.5);
                // float dis = distance(center, materialInput.st);
                // if(dis > 0.05){
                //     material.alpha = dis * 1.0;
                // } else {
                //     material.alpha = 1.0;
                // }
                return material;
            }
    `
    // videoElement.type = 'sampler2D';
    let m = new Material({
        fabric: {
            uniforms: {
                image: "",
                alpha: alpha
            },
            source: shaderSource
        }
    })
    m.uniforms.image = videoElement;
    // m.uniforms.color_2 = [1, 1, 1]
    // m.uniforms.repeat_1 = [1, 1]
    // let m = Material.fromType("Image", {
    //     uniforms: {
    //         image: videoElement
    //     }
    // })
    // m.uniforms.image = videoElement;
    // m.shaderSource = shaderSource
    await GroundPrimitive.initializeTerrainHeights()
    const primitive = new GroundPrimitive({
        asynchronous: false,
        geometryInstances: new GeometryInstance({
            geometry: new PolygonGeometry({
                polygonHierarchy: new PolygonHierarchy(controlPoints),
                vertexFormat: PolylineColorAppearance.VERTEX_FORMAT
            })
        }),
        appearance: new  MaterialAppearance({
            material: m,
            renderState: {
                blending: BlendingState.ALPHA_BLEND
            }
            // renderState: {
            // 	blending: BlendingState.PRE_MULTIPLIED_ALPHA_BLEND,
            // 	depthTest: { enabled: true },
            // 	depthMask: true,
            // },
        })
    })
    videoPrimitive.value = viewer.scene.primitives.add(primitive)
}

async function addPrimitiveVideo(controlPoints, alpha) {


    if (videoPrimitive.value) {
        viewer.scene.primitives.remove(videoPrimitive.value)
    }

    var material = Material.fromType('Image');
    material.uniforms.image = videoElement.value;
    // 视频
    // m.uniforms.image = videoElement.value;
    const primitive = new Primitive({
        asynchronous: false,
        geometryInstances: new GeometryInstance({
            geometry: new PolygonGeometry({

                perPositionHeight: true,
                polygonHierarchy: new PolygonHierarchy(controlPoints),
                // vertexFormat: PolylineColorAppearance.VERTEX_FORMAT
            })
        }),
        appearance: new MaterialAppearance({
            material: material,
            renderState: {
                blending: BlendingState.ALPHA_BLEND
            }
        })
    })
    videoPrimitive.value = viewer.scene.primitives.add(primitive)
}

async function addWallPrimitiveVideo(controlPoints, alpha) {

if (videoPrimitive.value) {
    viewer.scene.primitives.remove(videoPrimitive.value)
}

var material = Material.fromType('Image');
material.uniforms.image = videoElement.value;
// 视频
// m.uniforms.image = videoElement.value;
const primitive = new Primitive({
    asynchronous: false,
    geometryInstances: new GeometryInstance({
        geometry: new WallGeometry({
            positions: controlPoints,
            // vertexFormat: PolylineColorAppearance.VERTEX_FORMAT
        })
    }),
    appearance: new MaterialAppearance({
        material: material,
        renderState: {
            blending: BlendingState.ALPHA_BLEND
        }
    })
})
videoPrimitive.value = viewer.scene.primitives.add(primitive)
}



</script>
Copyright ©2018 DengQN
开往
十年之约
萌ICP备20240208号 异次元之旅
虫洞
SiteMap 站点地图
文章订阅 RSS
Powered by Nuxt3 & Vue3
管理 登录
Deploy