2017-07-28 08:32:08 +08:00
/*************************************************************************/
/* arkit_interface.mm */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
2019-08-28 15:19:52 +08:00
/* https://godotengine.org */
2017-07-28 08:32:08 +08:00
/*************************************************************************/
2020-01-01 18:16:22 +08:00
/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */
2017-07-28 08:32:08 +08:00
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
2020-04-28 21:19:37 +08:00
#include "core/input/input.h"
2017-07-28 08:32:08 +08:00
#include "core/os/os.h"
#include "scene/resources/surface_tool.h"
2020-03-28 08:50:00 +08:00
#include "servers/rendering/rendering_server_globals.h"
2017-07-28 08:32:08 +08:00
#import <ARKit/ARKit.h>
#import <UIKit/UIKit.h>
2019-11-20 17:18:01 +08:00
#include <dlfcn.h>
2017-07-28 08:32:08 +08:00
#include "arkit_interface.h"
#include "arkit_session_delegate.h"
// just a dirty workaround for now, declare these as globals. I'll probably encapsulate ARSession and associated logic into an mm object and change ARKitInterface to a normal cpp object that consumes it.
2020-07-16 00:03:01 +08:00
API_AVAILABLE(ios(11.0))
2017-07-28 08:32:08 +08:00
ARSession *ar_session;
2020-07-16 00:03:01 +08:00
2017-07-28 08:32:08 +08:00
ARKitSessionDelegate *ar_delegate;
NSTimeInterval last_timestamp;
/* this is called when we initialize or when we come back from having our app pushed to the background, just (re)start our session */
void ARKitInterface::start_session() {
// We're active...
session_was_started = true;
// Ignore this if we're not initialized...
if (initialized) {
print_line("Starting ARKit session");
2019-11-20 17:18:01 +08:00
2020-07-16 00:03:01 +08:00
if (@available(iOS 11, *)) {
Class ARWorldTrackingConfigurationClass = NSClassFromString(@"ARWorldTrackingConfiguration");
ARWorldTrackingConfiguration *configuration = [ARWorldTrackingConfigurationClass new];
2019-11-20 17:18:01 +08:00
2020-07-16 00:03:01 +08:00
configuration.lightEstimationEnabled = light_estimation_is_enabled;
if (plane_detection_is_enabled) {
if (@available(iOS 11.3, *)) {
configuration.planeDetection = ARPlaneDetectionVertical | ARPlaneDetectionHorizontal;
} else {
configuration.planeDetection = ARPlaneDetectionHorizontal;
}
} else {
configuration.planeDetection = 0;
}
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
// make sure our camera is on
if (feed.is_valid()) {
feed->set_active(true);
}
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
[ar_session runWithConfiguration:configuration];
}
2017-07-28 08:32:08 +08:00
}
}
void ARKitInterface::stop_session() {
session_was_started = false;
// Ignore this if we're not initialized...
if (initialized) {
// make sure our camera is off
if (feed.is_valid()) {
feed->set_active(false);
}
2020-07-16 00:03:01 +08:00
if (@available(iOS 11.0, *)) {
[ar_session pause];
}
2017-07-28 08:32:08 +08:00
}
}
void ARKitInterface::notification(int p_what) {
// TODO, this is not being called, need to find out why, possibly because this is not a node.
// in that case we need to find a way to get these notifications!
switch (p_what) {
2020-07-16 00:03:01 +08:00
case DisplayServer::WINDOW_EVENT_FOCUS_IN: {
2017-07-28 08:32:08 +08:00
print_line("Focus in");
start_session();
}; break;
2020-07-16 00:03:01 +08:00
case DisplayServer::WINDOW_EVENT_FOCUS_OUT: {
2017-07-28 08:32:08 +08:00
print_line("Focus out");
stop_session();
}; break;
default:
break;
}
}
bool ARKitInterface::get_anchor_detection_is_enabled() const {
return plane_detection_is_enabled;
}
void ARKitInterface::set_anchor_detection_is_enabled(bool p_enable) {
if (plane_detection_is_enabled != p_enable) {
plane_detection_is_enabled = p_enable;
// Restart our session (this will be ignore if we're not initialised)
if (session_was_started) {
start_session();
}
}
}
int ARKitInterface::get_camera_feed_id() {
if (feed.is_null()) {
return 0;
} else {
return feed->get_id();
}
}
bool ARKitInterface::get_light_estimation_is_enabled() const {
return light_estimation_is_enabled;
}
void ARKitInterface::set_light_estimation_is_enabled(bool p_enable) {
if (light_estimation_is_enabled != p_enable) {
light_estimation_is_enabled = p_enable;
// Restart our session (this will be ignore if we're not initialised)
if (session_was_started) {
start_session();
}
}
}
real_t ARKitInterface::get_ambient_intensity() const {
return ambient_intensity;
}
real_t ARKitInterface::get_ambient_color_temperature() const {
return ambient_color_temperature;
}
StringName ARKitInterface::get_name() const {
return "ARKit";
}
int ARKitInterface::get_capabilities() const {
2020-04-08 22:47:36 +08:00
return ARKitInterface::XR_MONO + ARKitInterface::XR_AR;
2017-07-28 08:32:08 +08:00
}
Array ARKitInterface::raycast(Vector2 p_screen_coord) {
2020-07-16 00:03:01 +08:00
if (@available(iOS 11, *)) {
Array arr;
Size2 screen_size = DisplayServer::get_singleton()->screen_get_size();
CGPoint point;
point.x = p_screen_coord.x / screen_size.x;
point.y = p_screen_coord.y / screen_size.y;
///@TODO maybe give more options here, for now we're taking just ARAchors into account that were found during plane detection keeping their size into account
NSArray<ARHitTestResult *> *results = [ar_session.currentFrame hitTest:point types:ARHitTestResultTypeExistingPlaneUsingExtent];
for (ARHitTestResult *result in results) {
Transform transform;
matrix_float4x4 m44 = result.worldTransform;
transform.basis.elements[0].x = m44.columns[0][0];
transform.basis.elements[1].x = m44.columns[0][1];
transform.basis.elements[2].x = m44.columns[0][2];
transform.basis.elements[0].y = m44.columns[1][0];
transform.basis.elements[1].y = m44.columns[1][1];
transform.basis.elements[2].y = m44.columns[1][2];
transform.basis.elements[0].z = m44.columns[2][0];
transform.basis.elements[1].z = m44.columns[2][1];
transform.basis.elements[2].z = m44.columns[2][2];
transform.origin.x = m44.columns[3][0];
transform.origin.y = m44.columns[3][1];
transform.origin.z = m44.columns[3][2];
/* important, NOT scaled to world_scale !! */
arr.push_back(transform);
}
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
return arr;
} else {
return Array();
}
2017-07-28 08:32:08 +08:00
}
void ARKitInterface::_bind_methods() {
ClassDB::bind_method(D_METHOD("_notification", "what"), &ARKitInterface::_notification);
ClassDB::bind_method(D_METHOD("set_light_estimation_is_enabled", "enable"), &ARKitInterface::set_light_estimation_is_enabled);
ClassDB::bind_method(D_METHOD("get_light_estimation_is_enabled"), &ARKitInterface::get_light_estimation_is_enabled);
ADD_PROPERTY(PropertyInfo(Variant::BOOL, "light_estimation"), "set_light_estimation_is_enabled", "get_light_estimation_is_enabled");
ClassDB::bind_method(D_METHOD("get_ambient_intensity"), &ARKitInterface::get_ambient_intensity);
ClassDB::bind_method(D_METHOD("get_ambient_color_temperature"), &ARKitInterface::get_ambient_color_temperature);
ClassDB::bind_method(D_METHOD("raycast", "screen_coord"), &ARKitInterface::raycast);
}
bool ARKitInterface::is_stereo() {
// this is a mono device...
return false;
}
bool ARKitInterface::is_initialized() const {
return initialized;
}
bool ARKitInterface::initialize() {
2020-04-08 22:47:36 +08:00
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL_V(xr_server, false);
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
if (@available(iOS 11, *)) {
if (!initialized) {
print_line("initializing ARKit");
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
// create our ar session and delegate
Class ARSessionClass = NSClassFromString(@"ARSession");
if (ARSessionClass == Nil) {
void *arkit_handle = dlopen("/System/Library/Frameworks/ARKit.framework/ARKit", RTLD_NOW);
if (arkit_handle) {
ARSessionClass = NSClassFromString(@"ARSession");
} else {
print_line("ARKit init failed");
return false;
}
2019-11-20 17:18:01 +08:00
}
2020-07-16 00:03:01 +08:00
ar_session = [ARSessionClass new];
ar_delegate = [ARKitSessionDelegate new];
ar_delegate.arkit_interface = this;
ar_session.delegate = ar_delegate;
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
// reset our transform
transform = Transform();
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
// make this our primary interface
xr_server->set_primary_interface(this);
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
// make sure we have our feed setup
if (feed.is_null()) {
feed.instance();
feed->set_name("ARKit");
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
CameraServer *cs = CameraServer::get_singleton();
if (cs != NULL) {
cs->add_feed(feed);
}
2017-07-28 08:32:08 +08:00
}
2020-07-16 00:03:01 +08:00
feed->set_active(true);
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
// yeah!
initialized = true;
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
// Start our session...
start_session();
}
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
return true;
} else {
return false;
}
2017-07-28 08:32:08 +08:00
}
void ARKitInterface::uninitialize() {
if (initialized) {
2020-04-08 22:47:36 +08:00
XRServer *xr_server = XRServer::get_singleton();
if (xr_server != NULL) {
2017-07-28 08:32:08 +08:00
// no longer our primary interface
2020-04-08 22:47:36 +08:00
xr_server->clear_primary_interface_if(this);
2017-07-28 08:32:08 +08:00
}
if (feed.is_valid()) {
CameraServer *cs = CameraServer::get_singleton();
if ((cs != NULL)) {
cs->remove_feed(feed);
}
feed.unref();
}
remove_all_anchors();
2020-07-16 00:03:01 +08:00
if (@available(iOS 11.0, *)) {
[ar_session release];
ar_session = NULL;
}
2017-07-28 08:32:08 +08:00
[ar_delegate release];
2020-07-16 00:03:01 +08:00
2017-07-28 08:32:08 +08:00
ar_delegate = NULL;
initialized = false;
session_was_started = false;
}
}
Size2 ARKitInterface::get_render_targetsize() {
2020-07-16 00:03:01 +08:00
// _THREAD_SAFE_METHOD_
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
Size2 target_size = DisplayServer::get_singleton()->screen_get_size();
2017-07-28 08:32:08 +08:00
return target_size;
}
2020-04-08 22:47:36 +08:00
Transform ARKitInterface::get_transform_for_eye(XRInterface::Eyes p_eye, const Transform &p_cam_transform) {
2020-07-16 00:03:01 +08:00
// _THREAD_SAFE_METHOD_
2017-07-28 08:32:08 +08:00
Transform transform_for_eye;
2020-04-08 22:47:36 +08:00
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL_V(xr_server, transform_for_eye);
2017-07-28 08:32:08 +08:00
if (initialized) {
2020-04-08 22:47:36 +08:00
float world_scale = xr_server->get_world_scale();
2017-07-28 08:32:08 +08:00
// just scale our origin point of our transform, note that we really shouldn't be using world_scale in ARKit but....
transform_for_eye = transform;
transform_for_eye.origin *= world_scale;
2020-04-08 22:47:36 +08:00
transform_for_eye = p_cam_transform * xr_server->get_reference_frame() * transform_for_eye;
2017-07-28 08:32:08 +08:00
} else {
// huh? well just return what we got....
transform_for_eye = p_cam_transform;
}
return transform_for_eye;
}
2020-04-08 22:47:36 +08:00
CameraMatrix ARKitInterface::get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) {
2017-07-28 08:32:08 +08:00
// Remember our near and far, it will be used in process when we obtain our projection from our ARKit session.
z_near = p_z_near;
z_far = p_z_far;
return projection;
}
2020-04-08 22:47:36 +08:00
void ARKitInterface::commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) {
2020-07-16 00:03:01 +08:00
// _THREAD_SAFE_METHOD_
2017-07-28 08:32:08 +08:00
// We must have a valid render target
ERR_FAIL_COND(!p_render_target.is_valid());
// Because we are rendering to our device we must use our main viewport!
ERR_FAIL_COND(p_screen_rect == Rect2());
// get the size of our screen
2020-07-16 00:03:01 +08:00
// Rect2 screen_rect = p_screen_rect;
2017-07-28 08:32:08 +08:00
// screen_rect.position.x += screen_rect.size.x;
// screen_rect.size.x = -screen_rect.size.x;
// screen_rect.position.y += screen_rect.size.y;
// screen_rect.size.y = -screen_rect.size.y;
2020-07-16 00:03:01 +08:00
// VSG::rasterizer->set_current_render_target(RID());
// VSG::rasterizer->blit_render_target_to_screen(p_render_target, screen_rect, 0);
2017-07-28 08:32:08 +08:00
}
2020-04-08 22:47:36 +08:00
XRPositionalTracker *ARKitInterface::get_anchor_for_uuid(const unsigned char *p_uuid) {
2017-07-28 08:32:08 +08:00
if (anchors == NULL) {
num_anchors = 0;
max_anchors = 10;
anchors = (anchor_map *)malloc(sizeof(anchor_map) * max_anchors);
}
ERR_FAIL_NULL_V(anchors, NULL);
for (unsigned int i = 0; i < num_anchors; i++) {
if (memcmp(anchors[i].uuid, p_uuid, 16) == 0) {
return anchors[i].tracker;
}
}
if (num_anchors + 1 == max_anchors) {
max_anchors += 10;
anchors = (anchor_map *)realloc(anchors, sizeof(anchor_map) * max_anchors);
ERR_FAIL_NULL_V(anchors, NULL);
}
2020-04-08 22:47:36 +08:00
XRPositionalTracker *new_tracker = memnew(XRPositionalTracker);
new_tracker->set_type(XRServer::TRACKER_ANCHOR);
2017-07-28 08:32:08 +08:00
char tracker_name[256];
sprintf(tracker_name, "Anchor %02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", p_uuid[0], p_uuid[1], p_uuid[2], p_uuid[3], p_uuid[4], p_uuid[5], p_uuid[6], p_uuid[7], p_uuid[8], p_uuid[9], p_uuid[10], p_uuid[11], p_uuid[12], p_uuid[13], p_uuid[14], p_uuid[15]);
String name = tracker_name;
print_line("Adding tracker " + name);
new_tracker->set_name(name);
// add our tracker
2020-04-08 22:47:36 +08:00
XRServer::get_singleton()->add_tracker(new_tracker);
2017-07-28 08:32:08 +08:00
anchors[num_anchors].tracker = new_tracker;
memcpy(anchors[num_anchors].uuid, p_uuid, 16);
num_anchors++;
return new_tracker;
}
void ARKitInterface::remove_anchor_for_uuid(const unsigned char *p_uuid) {
if (anchors != NULL) {
for (unsigned int i = 0; i < num_anchors; i++) {
if (memcmp(anchors[i].uuid, p_uuid, 16) == 0) {
// remove our tracker
2020-04-08 22:47:36 +08:00
XRServer::get_singleton()->remove_tracker(anchors[i].tracker);
2017-07-28 08:32:08 +08:00
memdelete(anchors[i].tracker);
// bring remaining forward
for (unsigned int j = i + 1; j < num_anchors; j++) {
anchors[j - 1] = anchors[j];
};
// decrease count
num_anchors--;
return;
}
}
}
}
void ARKitInterface::remove_all_anchors() {
if (anchors != NULL) {
for (unsigned int i = 0; i < num_anchors; i++) {
// remove our tracker
2020-04-08 22:47:36 +08:00
XRServer::get_singleton()->remove_tracker(anchors[i].tracker);
2017-07-28 08:32:08 +08:00
memdelete(anchors[i].tracker);
};
free(anchors);
anchors = NULL;
num_anchors = 0;
}
}
void ARKitInterface::process() {
2020-07-16 00:03:01 +08:00
// _THREAD_SAFE_METHOD_
2017-07-28 08:32:08 +08:00
if (@available(iOS 11.0, *)) {
if (initialized) {
// get our next ARFrame
ARFrame *current_frame = ar_session.currentFrame;
if (last_timestamp != current_frame.timestamp) {
// only process if we have a new frame
last_timestamp = current_frame.timestamp;
// get some info about our screen and orientation
2020-07-16 00:03:01 +08:00
Size2 screen_size = DisplayServer::get_singleton()->screen_get_size();
UIInterfaceOrientation orientation = UIInterfaceOrientationUnknown;
if (@available(iOS 13, *)) {
orientation = [UIApplication sharedApplication].delegate.window.windowScene.interfaceOrientation;
#if !defined(TARGET_OS_SIMULATOR) || !TARGET_OS_SIMULATOR
} else {
orientation = [[UIApplication sharedApplication] statusBarOrientation];
#endif
}
2017-07-28 08:32:08 +08:00
// Grab our camera image for our backbuffer
CVPixelBufferRef pixelBuffer = current_frame.capturedImage;
if ((CVPixelBufferGetPlaneCount(pixelBuffer) == 2) && (feed != NULL)) {
// Plane 0 is our Y and Plane 1 is our CbCr buffer
// ignored, we check each plane separately
// image_width = CVPixelBufferGetWidth(pixelBuffer);
// image_height = CVPixelBufferGetHeight(pixelBuffer);
// printf("Pixel buffer %i - %i\n", image_width, image_height);
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
// get our buffers
unsigned char *dataY = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
unsigned char *dataCbCr = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
if (dataY == NULL) {
print_line("Couldn't access Y pixel buffer data");
} else if (dataCbCr == NULL) {
print_line("Couldn't access CbCr pixel buffer data");
} else {
Ref<Image> img[2];
size_t extraLeft, extraRight, extraTop, extraBottom;
CVPixelBufferGetExtendedPixels(pixelBuffer, &extraLeft, &extraRight, &extraTop, &extraBottom);
{
// do Y
2020-07-16 00:03:01 +08:00
size_t new_width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
size_t new_height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
size_t bytes_per_row = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
2017-07-28 08:32:08 +08:00
if ((image_width[0] != new_width) || (image_height[0] != new_height)) {
printf("- Camera padding l:%lu r:%lu t:%lu b:%lu\n", extraLeft, extraRight, extraTop, extraBottom);
2020-07-16 00:03:01 +08:00
printf("- Camera Y plane size: %lu, %lu - %lu\n", new_width, new_height, bytes_per_row);
2017-07-28 08:32:08 +08:00
image_width[0] = new_width;
image_height[0] = new_height;
img_data[0].resize(new_width * new_height);
}
2020-07-16 00:03:01 +08:00
uint8_t *w = img_data[0].ptrw();
2017-07-28 08:32:08 +08:00
if (new_width == bytes_per_row) {
2020-07-16 00:03:01 +08:00
memcpy(w, dataY, new_width * new_height);
2017-07-28 08:32:08 +08:00
} else {
2020-07-16 00:03:01 +08:00
size_t offset_a = 0;
size_t offset_b = extraLeft + (extraTop * bytes_per_row);
for (size_t r = 0; r < new_height; r++) {
memcpy(w + offset_a, dataY + offset_b, new_width);
2017-07-28 08:32:08 +08:00
offset_a += new_width;
offset_b += bytes_per_row;
}
}
img[0].instance();
img[0]->create(new_width, new_height, 0, Image::FORMAT_R8, img_data[0]);
}
{
// do CbCr
2020-07-16 00:03:01 +08:00
size_t new_width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1);
size_t new_height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1);
size_t bytes_per_row = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
2017-07-28 08:32:08 +08:00
if ((image_width[1] != new_width) || (image_height[1] != new_height)) {
2020-07-16 00:03:01 +08:00
printf("- Camera CbCr plane size: %lu, %lu - %lu\n", new_width, new_height, bytes_per_row);
2017-07-28 08:32:08 +08:00
image_width[1] = new_width;
image_height[1] = new_height;
img_data[1].resize(2 * new_width * new_height);
}
2020-07-16 00:03:01 +08:00
uint8_t *w = img_data[1].ptrw();
2017-07-28 08:32:08 +08:00
if ((2 * new_width) == bytes_per_row) {
2020-07-16 00:03:01 +08:00
memcpy(w, dataCbCr, 2 * new_width * new_height);
2017-07-28 08:32:08 +08:00
} else {
2020-07-16 00:03:01 +08:00
size_t offset_a = 0;
size_t offset_b = extraLeft + (extraTop * bytes_per_row);
for (size_t r = 0; r < new_height; r++) {
memcpy(w + offset_a, dataCbCr + offset_b, 2 * new_width);
2017-07-28 08:32:08 +08:00
offset_a += 2 * new_width;
offset_b += bytes_per_row;
}
}
img[1].instance();
img[1]->create(new_width, new_height, 0, Image::FORMAT_RG8, img_data[1]);
}
// set our texture...
feed->set_YCbCr_imgs(img[0], img[1]);
// now build our transform to display this as a background image that matches our camera
CGAffineTransform affine_transform = [current_frame displayTransformForOrientation:orientation viewportSize:CGSizeMake(screen_size.width, screen_size.height)];
// we need to invert this, probably row v.s. column notation
affine_transform = CGAffineTransformInvert(affine_transform);
2019-07-23 02:22:03 +08:00
if (orientation != UIInterfaceOrientationPortrait) {
2017-07-28 08:32:08 +08:00
affine_transform.b = -affine_transform.b;
affine_transform.d = -affine_transform.d;
affine_transform.ty = 1.0 - affine_transform.ty;
} else {
affine_transform.c = -affine_transform.c;
affine_transform.a = -affine_transform.a;
affine_transform.tx = 1.0 - affine_transform.tx;
}
Transform2D display_transform = Transform2D(
affine_transform.a, affine_transform.b,
affine_transform.c, affine_transform.d,
affine_transform.tx, affine_transform.ty);
feed->set_transform(display_transform);
}
// and unlock
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
}
// Record light estimation to apply to our scene
if (light_estimation_is_enabled) {
ambient_intensity = current_frame.lightEstimate.ambientIntensity;
///@TODO it's there, but not there.. what to do with this...
// https://developer.apple.com/documentation/arkit/arlightestimate?language=objc
// ambient_color_temperature = current_frame.lightEstimate.ambientColorTemperature;
}
// Process our camera
ARCamera *camera = current_frame.camera;
// strangely enough we have to states, rolling them up into one
if (camera.trackingState == ARTrackingStateNotAvailable) {
// no tracking, would be good if we black out the screen or something...
2020-04-08 22:47:36 +08:00
tracking_state = XRInterface::XR_NOT_TRACKING;
2017-07-28 08:32:08 +08:00
} else {
if (camera.trackingState == ARTrackingStateNormal) {
2020-04-08 22:47:36 +08:00
tracking_state = XRInterface::XR_NORMAL_TRACKING;
2017-07-28 08:32:08 +08:00
} else if (camera.trackingStateReason == ARTrackingStateReasonExcessiveMotion) {
2020-04-08 22:47:36 +08:00
tracking_state = XRInterface::XR_EXCESSIVE_MOTION;
2017-07-28 08:32:08 +08:00
} else if (camera.trackingStateReason == ARTrackingStateReasonInsufficientFeatures) {
2020-04-08 22:47:36 +08:00
tracking_state = XRInterface::XR_INSUFFICIENT_FEATURES;
2017-07-28 08:32:08 +08:00
} else {
2020-04-08 22:47:36 +08:00
tracking_state = XRInterface::XR_UNKNOWN_TRACKING;
2017-07-28 08:32:08 +08:00
}
// copy our current frame transform
matrix_float4x4 m44 = camera.transform;
2019-07-23 02:22:03 +08:00
if (orientation == UIInterfaceOrientationLandscapeLeft) {
2017-07-28 08:32:08 +08:00
transform.basis.elements[0].x = m44.columns[0][0];
transform.basis.elements[1].x = m44.columns[0][1];
transform.basis.elements[2].x = m44.columns[0][2];
transform.basis.elements[0].y = m44.columns[1][0];
transform.basis.elements[1].y = m44.columns[1][1];
transform.basis.elements[2].y = m44.columns[1][2];
2019-07-23 02:22:03 +08:00
} else if (orientation == UIInterfaceOrientationPortrait) {
2017-07-28 08:32:08 +08:00
transform.basis.elements[0].x = m44.columns[1][0];
transform.basis.elements[1].x = m44.columns[1][1];
transform.basis.elements[2].x = m44.columns[1][2];
transform.basis.elements[0].y = -m44.columns[0][0];
transform.basis.elements[1].y = -m44.columns[0][1];
transform.basis.elements[2].y = -m44.columns[0][2];
2019-07-23 02:22:03 +08:00
} else if (orientation == UIInterfaceOrientationLandscapeRight) {
2017-07-28 08:32:08 +08:00
transform.basis.elements[0].x = -m44.columns[0][0];
transform.basis.elements[1].x = -m44.columns[0][1];
transform.basis.elements[2].x = -m44.columns[0][2];
transform.basis.elements[0].y = -m44.columns[1][0];
transform.basis.elements[1].y = -m44.columns[1][1];
transform.basis.elements[2].y = -m44.columns[1][2];
2019-07-23 02:22:03 +08:00
} else if (orientation == UIInterfaceOrientationPortraitUpsideDown) {
2017-07-28 08:32:08 +08:00
// this may not be correct
transform.basis.elements[0].x = m44.columns[1][0];
transform.basis.elements[1].x = m44.columns[1][1];
transform.basis.elements[2].x = m44.columns[1][2];
transform.basis.elements[0].y = m44.columns[0][0];
transform.basis.elements[1].y = m44.columns[0][1];
transform.basis.elements[2].y = m44.columns[0][2];
}
transform.basis.elements[0].z = m44.columns[2][0];
transform.basis.elements[1].z = m44.columns[2][1];
transform.basis.elements[2].z = m44.columns[2][2];
transform.origin.x = m44.columns[3][0];
transform.origin.y = m44.columns[3][1];
transform.origin.z = m44.columns[3][2];
// copy our current frame projection, investigate using projectionMatrixWithViewportSize:orientation:zNear:zFar: so we can set our own near and far
m44 = [camera projectionMatrixForOrientation:orientation viewportSize:CGSizeMake(screen_size.width, screen_size.height) zNear:z_near zFar:z_far];
projection.matrix[0][0] = m44.columns[0][0];
projection.matrix[1][0] = m44.columns[1][0];
projection.matrix[2][0] = m44.columns[2][0];
projection.matrix[3][0] = m44.columns[3][0];
projection.matrix[0][1] = m44.columns[0][1];
projection.matrix[1][1] = m44.columns[1][1];
projection.matrix[2][1] = m44.columns[2][1];
projection.matrix[3][1] = m44.columns[3][1];
projection.matrix[0][2] = m44.columns[0][2];
projection.matrix[1][2] = m44.columns[1][2];
projection.matrix[2][2] = m44.columns[2][2];
projection.matrix[3][2] = m44.columns[3][2];
projection.matrix[0][3] = m44.columns[0][3];
projection.matrix[1][3] = m44.columns[1][3];
projection.matrix[2][3] = m44.columns[2][3];
projection.matrix[3][3] = m44.columns[3][3];
}
}
}
}
}
void ARKitInterface::_add_or_update_anchor(void *p_anchor) {
2020-07-16 00:03:01 +08:00
// _THREAD_SAFE_METHOD_
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
if (@available(iOS 11.0, *)) {
ARAnchor *anchor = (ARAnchor *)p_anchor;
unsigned char uuid[16];
[anchor.identifier getUUIDBytes:uuid];
XRPositionalTracker *tracker = get_anchor_for_uuid(uuid);
if (tracker != NULL) {
// lets update our mesh! (using Arjens code as is for now)
// we should also probably limit how often we do this...
// can we safely cast this?
ARPlaneAnchor *planeAnchor = (ARPlaneAnchor *)anchor;
if (@available(iOS 11.3, *)) {
if (planeAnchor.geometry.triangleCount > 0) {
Ref<SurfaceTool> surftool;
surftool.instance();
surftool->begin(Mesh::PRIMITIVE_TRIANGLES);
for (int j = planeAnchor.geometry.triangleCount * 3 - 1; j >= 0; j--) {
int16_t index = planeAnchor.geometry.triangleIndices[j];
simd_float3 vrtx = planeAnchor.geometry.vertices[index];
simd_float2 textcoord = planeAnchor.geometry.textureCoordinates[index];
surftool->add_uv(Vector2(textcoord[0], textcoord[1]));
surftool->add_color(Color(0.8, 0.8, 0.8));
surftool->add_vertex(Vector3(vrtx[0], vrtx[1], vrtx[2]));
}
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
surftool->generate_normals();
tracker->set_mesh(surftool->commit());
} else {
Ref<Mesh> nomesh;
tracker->set_mesh(nomesh);
}
} else {
Ref<Mesh> nomesh;
tracker->set_mesh(nomesh);
2017-07-28 08:32:08 +08:00
}
2020-07-16 00:03:01 +08:00
// Note, this also contains a scale factor which gives us an idea of the size of the anchor
// We may extract that in our XRAnchor class
Basis b;
matrix_float4x4 m44 = anchor.transform;
b.elements[0].x = m44.columns[0][0];
b.elements[1].x = m44.columns[0][1];
b.elements[2].x = m44.columns[0][2];
b.elements[0].y = m44.columns[1][0];
b.elements[1].y = m44.columns[1][1];
b.elements[2].y = m44.columns[1][2];
b.elements[0].z = m44.columns[2][0];
b.elements[1].z = m44.columns[2][1];
b.elements[2].z = m44.columns[2][2];
tracker->set_orientation(b);
tracker->set_rw_position(Vector3(m44.columns[3][0], m44.columns[3][1], m44.columns[3][2]));
2017-07-28 08:32:08 +08:00
}
}
}
void ARKitInterface::_remove_anchor(void *p_anchor) {
2020-07-16 00:03:01 +08:00
// _THREAD_SAFE_METHOD_
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
if (@available(iOS 11.0, *)) {
ARAnchor *anchor = (ARAnchor *)p_anchor;
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
unsigned char uuid[16];
[anchor.identifier getUUIDBytes:uuid];
2017-07-28 08:32:08 +08:00
2020-07-16 00:03:01 +08:00
remove_anchor_for_uuid(uuid);
}
2017-07-28 08:32:08 +08:00
}
ARKitInterface::ARKitInterface() {
initialized = false;
session_was_started = false;
plane_detection_is_enabled = false;
light_estimation_is_enabled = false;
2020-07-16 00:03:01 +08:00
if (@available(iOS 11.0, *)) {
ar_session = NULL;
}
2017-07-28 08:32:08 +08:00
z_near = 0.01;
z_far = 1000.0;
projection.set_perspective(60.0, 1.0, z_near, z_far, false);
anchors = NULL;
num_anchors = 0;
ambient_intensity = 1.0;
ambient_color_temperature = 1.0;
image_width[0] = 0;
image_width[1] = 0;
image_height[0] = 0;
image_height[1] = 0;
}
ARKitInterface::~ARKitInterface() {
remove_all_anchors();
// and make sure we cleanup if we haven't already
if (is_initialized()) {
uninitialize();
}
}
