Wave MR Marker Native Development Guide

Overview

To develop MR applications using Wave SDK, please refer to following documents:

Contents 1. Manifest 2. Show passthrouh 3. Init Marker 4. Detect Marker 5. Track Marker

Note

• You can download the Wave SDK from here. And check Native marker sample in SDK/samples/wvr_native_marker.
• To build the project, please reference Native SDK Getting Started.

1. Manifest

Developer should add feature in AndroidManifest.xml.

//andoirManifest.xml
<uses-feature android:name="wave.feature.marker" android:required="true"/>

2. Show passthrouh

Show Passthrough underlay by calling WVR_ShowPassthroughUnderlay() after WVR_Init().

static bool showUnderlay = true;
WVR_ShowPassthroughUnderlay(showUnderlay);

3. Init Marker

Application should Call WVR_StartMarker() to initiate marker feature. After initialization is done, developer should select which type of marker will be observed in this application.

bool MarkerManager::startMarker(){
    if (WVR_StartMarker() != WVR_Success)
        LOGE("WVR_StartMarker fail");
    if (WVR_StartMarkerObserver(WVR_MarkerObserverTarget_Aruco) != WVR_Success)
        LOGE("WVR_StartMarkerObserver fail");
    return true;
}

4. Detect Marker

4-1. Start detecting marker - WVR_StartMarkerDetection()

In order to detecting environment markers, application should change the observer state to WVR_MarkerObserveState_Detecting by calling WVR_StartMarkerDetection() first. Observer state can be checked by calling WVR_GetMarkerObserverState().

bool MarkerManager::tryDetectingArucoMarker()
{
    WVR_MarkerObserverState state;

    // check observer state
    WVR_GetMarkerObserverState(WVR_MarkerObserverTarget_Aruco, &state);

    switch (state)
    {
    case WVR_MarkerObserverState_Detecting:
        LOGI("Observe state already WVR_MarkerObserverState_Detecting");
        break;
    case WVR_MarkerObserverState_Tracking:
        stopTrackingAllTrackableMarker();
        LOGI("Observe state WVR_MarkerObserverState_Tracking -> WVR_MarkerObserverState_Detecting");
        if (WVR_StartMarkerDetection(WVR_MarkerObserverTarget_Aruco))
            LOGE("marker init: markerAPI WVR_StartMarkerDetection fail");
        break;
    case WVR_MarkerObserverState_Idle:
        LOGI("Observe state WVR_MarkerObserverState_Idle -> WVR_MarkerObserverState_Detecting");
        if (WVR_StartMarkerDetection(WVR_MarkerObserverTarget_Aruco))
            LOGE("marker init: markerAPI WVR_StartMarkerDetection fail");
        break;
    }
    return true;
}

4-2. Get detected marker data every frame - WVR_GetArucoMarker()

Developer can call WVR_GetArucoMarker() twice to get the number of all detected markers and marker data after calling WVR_StartMarkerDetection().

std::vector<WVR_ArucoMarker> detectingArucoData;
void MarkerManager::getDetectingData()
{
    static uint32_t lastObserveSize=-1;
    uint32_t size = 0;
    detectingArucoData.clear();
    // get how many markers has been detected
    WVR_GetArucoMarkers(size, &size, WVR_PoseOriginModel_OriginOnHead, nullptr);
    // print log when size change
    if (size != lastObserveSize)
    {
        LOGI("WVR_GetArucoMarkers size=%d", size);
        lastObserveSize = size;
    }
    detectingArucoData.resize(size);
    WVR_GetArucoMarkers(size, &size, WVR_PoseOriginModel_OriginOnHead, detectingArucoData.data());
}

4-3. Render Detected marker

Developer should notice that when observer state is WVR_MarkerObserverState_Detecting, detected marker have two type of state, which are WVR_MarkerTrackingState_Detected and WVR_MarkerTrackingState_Stopped. WVR_ArudoMarker.state would be WVR_MarkerTrackingState_Stopped if WVR_ArudoMarker.trackId had been created as a trackable marker by WVR_CreateTrackableMarker() before.

void PlaneManager::render(const Matrix4 &projection, const Matrix4 &eye, const Matrix4 &view,
    std::vector<WVR_ArucoMarker> &markerData)
{
    mShader->useProgram();
    std::vector<Matrix4> matrix;
    for (int a = 0; a < markerData.size(); a++)
    {
        if (markerData[a].state == WVR_MarkerTrackingState_Detected)
        {
            // white (detected)
            glUniform4f(mColor, 0.9f, 0.9f, 0.9f, 0.7f);
        }
        else if (markerData[a].state == WVR_MarkerTrackingState_Stopped)
        {
            LOGI("markerData [a]=WVR_MarkerTrackingState_Stopped", a);
                // green (which was already created as trackable marker)
                glUniform4f(mColor, 0.3f, 0.9f, 0.3f, 0.7f);
        }
        else
        {
            // unexpected status
            continue;
        }

        matrix.push_back(projection * eye * view * convertToMatrix4(markerData[a].pose));
        glUniformMatrix4fv(mMatrixLocation, 1, false, matrix[a].get());
        glUniform3f(mScale, markerData[a].size, markerData[a].size, markerData[a].size);
        Matrix4 mmt = convertToMatrix4(markerData[a].pose);

        mVAO->bindVAO();
        glDrawElements(GL_TRIANGLES, singleIndexArray.size(), GL_UNSIGNED_INT, 0);
        mVAO->unbindVAO();
    }
    mShader->unuseProgram();
    renderAxes(matrix);
}

mPlanes->render(mProjectionLeft, mEyePosLeft, mHMDPose, mMarkerManager->detectingArucoData);
mPlanes->render(mProjectionRight, mEyePosRight, mHMDPose, mMarkerManager->detectingArucoData);
mScanLine->render();

5. Track Marker

5-1. Create trackable marker - WVR_CreateTrackableMarker()

In order to track specific marker in the following chapter, developer should create specific marker as a trackable marker by calling WVR_CreateTrackableMarker() first.

bool MarkerManager::addTrackableMarker(int selectIndex)
{
    char name[] = "helloMarker\0";
    WVR_MarkerName markerName;
    strncpy(markerName.name, name, sizeof(name) / sizeof(char));
    WVR_TrackableMarkerCreateInfo createInfo{detectingArucoData[selectIndex].uuid, markerName};

    // All created trackableMarker WVR_MarkerTrackingState status will be WVR_MarkerTrackingState_Stopped
    // while observer state == WVR_MarkerObserverState_Detecting
    WVR_CreateTrackableMarker(&createInfo);
    return true;
}

5-2. Delete trackable marker- WVR_DestroyTrackableMarker ()

Developer can also delete trackable marker by calling WVR_DestroyTrackableMarker()

std::vector<WVR_ArucoMarker> detectingArucoData;

bool MarkerManager::deleteTrackableMarker(int selectIndex)
{
    WVR_DestroyTrackableMarker(detectingArucoData[selectIndex].uuid);
    return true;
}

5-3. Enumerate all trackable marker - WVR_EnumerateTrackableMarkers()

Developer can enumerate all trackable markers which were created(WVR_CreateTrackableMarker()) by calling WVR_EnumerateTrackableMarkers(). We recommend that developer should call this function every time after adding trackable marker, deleting trackable marker, observer state change and application resume.

std::vector<WVR_Uuid> trackableIds;

void MarkerManager::refreshTrackableMarker()
{
    uint32_t size = 0;
    trackableIds.clear();
    WVR_EnumerateTrackableMarkers(WVR_MarkerObserverTarget_Aruco, size, &size, nullptr);
    trackableIds.resize(size);
    WVR_EnumerateTrackableMarkers(WVR_MarkerObserverTarget_Aruco, size, &size, trackableIds.data());
    LOGI("WVR_EnumerateTrackableMarkers get size=%d", size);
    filterTrackableAruco();
}

5-4. Check trackable marker state - WVR_GetTrackableMarkerState ()

Developer can check marker state by calling WVR_GetTrackableMarkerState(). In our sample code, we are only interested in aruco marker. Therefore, we will check whether WVR_TrackableMarkerState.Target is equal to WVR_MarkerObserverTarget_Aruco.

typedef struct mTrackableArucoData
{
    WVR_Uuid uuid;
    WVR_TrackableMarkerState markerState;
    WVR_ArucoMarkerData arucoData;
} mTrackableArucoData;
std::vector<mTrackableArucoData> trackableMarkerData;

void MarkerManager::filterTrackableAruco()
{
    //  get trackable aruco marker data to render
    trackableMarkerData.clear();
    WVR_TrackableMarkerState tempState;
    WVR_ArucoMarkerData tempData;
    for (int a = 0; a < trackableIds.size(); a++)
    {
        WVR_GetTrackableMarkerState(trackableIds[a], WVR_PoseOriginModel_OriginOnHead, &tempState);

        if (tempState.target == WVR_MarkerObserverTarget_Aruco)
        {
            WVR_GetArucoMarkerData(trackableIds[a], &tempData);
            trackableMarkerData.push_back(mTrackableArucoData{
                .uuid = trackableIds[a],
                .markerState = tempState,
                .arucoData = tempData});
        }
    }
}

5-5. Start tracking trackable marker- WVR_StartTrackableMarkerTracking ()

Developer can start tracking specific marker by sending WVR_Uuid(Get from WVR_EnumerateTrackableMarkers()) to WVR_StartTrackableMarkerTracking(). Developer should aware that the observer state will be change to WVR_MarkerObserverState_Tracking automatically, right after you successfully start tracking a trackable marker.

typedef struct mTrackableArucoData
{
    WVR_Uuid uuid;
    WVR_TrackableMarkerState markerState;
    WVR_ArucoMarkerData arucoData;
} mTrackableArucoData;
std::vector<mTrackableArucoData> trackableMarkerData;

bool MarkerManager::startTrackingAllTrackableMarker()
{
    refreshTrackableMarker();
    if (trackableMarkerData.size() == 0)
        return false;

    // start tracking all trackable aruco markers
    for (int a = 0; a < trackableMarkerData.size(); a++)
    {
        // observer state will change to WVR_MarkerObserverState_Tracking
        if (WVR_StartTrackableMarkerTracking(trackableMarkerData[a].uuid) != WVR_Success)
            LOGE("markerAPI WVR_StartTrackableMarkerTracking marker %d fail", a);
    }
    return true;
}

5-6. Stop tracking trackable marker- WVR_StopTrackableMarkerTracking ()

Developer can stop tracking certain trackable maker by sending WVR_Uuid(Get from WVR_EnumerateTrackableMarkers()) to WVR_StopTrackableMarkerTracking(). When all tracking marker been stopped, observer state will change from WVR_MarkerObserverState_Tracking to WVR_MarkerObserverState_Idle automatically.

std::vector<WVR_Uuid> trackableIds;

bool MarkerManager::stopTrackingTrackableMarker(int selectIndex)
{
    LOGI("stop trackable marker at [%d]", selectIndex);
    // when all trackable marker had been stopped, observer state will change
    // from WVR_MarkerObserverState_Tracking to WVR_MarkerObserverState_Idle
    WVR_StopTrackableMarkerTracking(trackableIds[selectIndex]);
    refreshTrackableMarker();
    return false;
}

5-7. Get trackable marker data every frame - WVR_GetTrackableMarkerState()/WVR_GetArucoMarkerData()

In order to get necessity data for rendering, develop should Call WVR_GetTrackableMarkerState() every frame to update marker pose and state. Futhermore, if the maker is aruco marker, develop could also call WVR_GetArucoMarkerData() to update size and trackerID

typedef struct mTrackableArucoData
{
    WVR_Uuid uuid;
    WVR_TrackableMarkerState markerState;
    WVR_ArucoMarkerData arucoData;
} mTrackableArucoData;
std::vector<mTrackableArucoData> trackableMarkerData;

void MarkerManager::getTrackingData()
{
    for (int a = 0; a < trackableMarkerData.size(); a++)
    {
        // update marker data
        WVR_GetTrackableMarkerState(trackableMarkerData[a].uuid, WVR_PoseOriginModel_OriginOnHead, &trackableMarkerData[a].markerState);
        // update aruco marker size
        WVR_GetArucoMarkerData(trackableMarkerData[a].uuid, &trackableMarkerData[a].arucoData);
    }
}

5-8. Render trackable marker

Render trackable marker by the pose and size we get form WVR_GetTrackableMarkerState() and WVR_GetArucoMarkerData(). In our sample, we will not render lost tracking marker(WVR_MarkerTrackingState_Paused) and the marker which had been stopped(WVR_MarkerTrackingState_Stopped).

void CubeManager::renderCubes(Matrix4 projection, Matrix4 eye, Matrix4 view, std::vector<mTrackableArucoData> &markerData)
{
    mShader->useProgram();
    for (int a = 0; a < markerData.size(); a++)
    {
        // skip lost tracking anchor
        if (markerData[a].markerState.state != WVR_MarkerTrackingState_Tracked)
        {
            LOGI("markerData %d is lost tracking", a);
            continue;
        }

        Matrix4 matrix = projection * eye * view * convertToMatrix4(markerData[a].markerState.pose);

        glUniformMatrix4fv(mMatrixLocation, 1, false, matrix.get());
        if (leftSelectCandidate.size() > 0 && nearestLeftSelect == a)
        {
            // red cube: can be stopped
            glUniform1i(mUseInput, 1);
            glUniform4f(mCubeColor, 1.0f, 0.0f, 0.0f, 0.4f);
        }
        else
        {
            // cube with texture
            glUniform1i(mUseInput, 0);
        }
        glUniform3f(mScale, markerData[a].arucoData.size, markerData[a].arucoData.size, markerData[a].arucoData.size);

        mVAO->bindVAO();
        glActiveTexture(GL_TEXTURE0);
        mTexture->bindTexture();
        glDrawElements(GL_TRIANGLES, singleIndexArray.size(), GL_UNSIGNED_INT, 0);
        mTexture->unbindTexture();
        mVAO->unbindVAO();
    }
    mShader->unuseProgram();
}