This project is read-only.

Project Description

Real Square is a mapping application for Windows Phone, which can show nearby locations on the map.

Like any map and navigation application. Also when the user hold the phone up right there come the difference and user will experience an augmented reality presentation of nearby locations. Each location will be tagged in Virtual 3D world and if the phone’s camera is on a location, the tag will shown over that location in 3D Virtual Reality. It is on the windows phone market now!
Click to try this experience in windows phone.

Future Enhancement Plans

First of all we want to port this project from Windows Mobile 7 to Windows Mobile 8.0 and 8.1. Good developers at 3d development, XNA, DirectX, Unity etc... needed.

Also, We want to put this augmented reality framework to the NuGet as an plug in play windows phone augmented reality DLL. So developers can integrate this project to their windows phone solutions.

Screenshots

IMAGE_154.jpg
IMAGE_156.jpg
capturedImage (16).jpg
capturedImage (10).png
capturedImage (14).png
capturedImage (15).jpg

Details of Development

What is Augmented reality

Augmented reality is a live direct or indirect view of a physical, real-world environment whose elements are augmented by computer-generated sensory input such as sound, video, graphics or GPS data. Wikipedia

This project is also a kind of representation and demo project for augmented reality in windows phone environment. To create Augmented Reality Experience in Windows Phone environment XNA should be used.

First of all we could create a basic shape. This shape has 6 faces but to create a place mark we removed 5 faces because we need just the front face.
using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Graphics;

namespace ARFinity.XnaObjects
{
    internal class BasicShape
    {
        public Vector3 shapeSize;
        public Vector3 shapePosition;
        private VertexPositionNormalTexture[] shapeVertices;
        private int shapeTriangles;
        private VertexBuffer shapeBuffer;
        public Texture2D shapeTexture;        

        public BasicShape(Vector3 size, Vector3 position)
        {
            shapeSize = size;
            shapePosition = position;
        }

        private void BuildShape()
        {
            shapeTriangles = 12;

            shapeVertices = new VertexPositionNormalTexture[36];

            Vector3 topLeftFront = shapePosition + new Vector3(-1.0f, 1.0f, -1.0f) * shapeSize;
            Vector3 bottomLeftFront = shapePosition + new Vector3(-1.0f, -1.0f, -1.0f) * shapeSize;
            Vector3 topRightFront = shapePosition + new Vector3(1.0f, 1.0f, -1.0f) * shapeSize;
            Vector3 bottomRightFront = shapePosition + new Vector3(1.0f, -1.0f, -1.0f) * shapeSize;

            Vector3 topLeftBack = shapePosition + new Vector3(-1.0f, 1.0f, 1.0f) * shapeSize;
            Vector3 topRightBack = shapePosition + new Vector3(1.0f, 1.0f, 1.0f) * shapeSize;
            Vector3 bottomLeftBack = shapePosition + new Vector3(-1.0f, -1.0f, 1.0f) * shapeSize;
            Vector3 bottomRightBack = shapePosition + new Vector3(1.0f, -1.0f, 1.0f) * shapeSize;

            Vector3 frontNormal = new Vector3(0.0f, 0.0f, 1.0f) * shapeSize;
            Vector3 backNormal = new Vector3(0.0f, 0.0f, -1.0f) * shapeSize;
            Vector3 topNormal = new Vector3(0.0f, 1.0f, 0.0f) * shapeSize;
            Vector3 bottomNormal = new Vector3(0.0f, -1.0f, 0.0f) * shapeSize;
            Vector3 leftNormal = new Vector3(-1.0f, 0.0f, 0.0f) * shapeSize;
            Vector3 rightNormal = new Vector3(1.0f, 0.0f, 0.0f) * shapeSize;

            //Vector2 textureTopLeft = new Vector2(0.5f * shapeSize.X, 0.0f * shapeSize.Y);
            //Vector2 textureTopRight = new Vector2(0.0f * shapeSize.X, 0.0f * shapeSize.Y);
            //Vector2 textureBottomLeft = new Vector2(0.5f * shapeSize.X, 0.5f * shapeSize.Y);
            //Vector2 textureBottomRight = new Vector2(0.0f * shapeSize.X, 0.5f * shapeSize.Y);

            Vector2 textureTopLeft = new Vector2(1.0f, 0.0f);
            Vector2 textureTopRight = new Vector2(0.0f, 0.0f);
            Vector2 textureBottomLeft = new Vector2(1.0f, 1.0f);
            Vector2 textureBottomRight = new Vector2(0.0f, 1.0f);

            // Front face.
            shapeVertices[0] = new VertexPositionNormalTexture(topLeftFront, frontNormal, textureTopLeft);
            shapeVertices[1] = new VertexPositionNormalTexture(bottomLeftFront, frontNormal, textureBottomLeft);
            shapeVertices[2] = new VertexPositionNormalTexture(topRightFront, frontNormal, textureTopRight);
            shapeVertices[3] = new VertexPositionNormalTexture(bottomLeftFront, frontNormal, textureBottomLeft);
            shapeVertices[4] = new VertexPositionNormalTexture(bottomRightFront, frontNormal, textureBottomRight);
            shapeVertices[5] = new VertexPositionNormalTexture(topRightFront, frontNormal, textureTopRight);


            //// Back face.
            //shapeVertices[6] = new VertexPositionNormalTexture(topLeftBack, backNormal, textureTopRight);
            //shapeVertices[7] = new VertexPositionNormalTexture(topRightBack, backNormal, textureTopLeft);
            //shapeVertices[8] = new VertexPositionNormalTexture(bottomLeftBack, backNormal, textureBottomRight);
            //shapeVertices[9] = new VertexPositionNormalTexture(bottomLeftBack, backNormal, textureBottomRight);
            //shapeVertices[10] = new VertexPositionNormalTexture(topRightBack, backNormal, textureTopLeft);
            //shapeVertices[11] = new VertexPositionNormalTexture(bottomRightBack, backNormal, textureBottomLeft);

            //// Top face.
            //shapeVertices[12] = new VertexPositionNormalTexture(topLeftFront, topNormal, textureBottomLeft);
            //shapeVertices[13] = new VertexPositionNormalTexture(topRightBack, topNormal, textureTopRight);
            //shapeVertices[14] = new VertexPositionNormalTexture(topLeftBack, topNormal, textureTopLeft);
            //shapeVertices[15] = new VertexPositionNormalTexture(topLeftFront, topNormal, textureBottomLeft);
            //shapeVertices[16] = new VertexPositionNormalTexture(topRightFront, topNormal, textureBottomRight);
            //shapeVertices[17] = new VertexPositionNormalTexture(topRightBack, topNormal, textureTopRight);

            //// Bottom face. 
            //shapeVertices[18] = new VertexPositionNormalTexture(bottomLeftFront, bottomNormal, textureTopLeft);
            //shapeVertices[19] = new VertexPositionNormalTexture(bottomLeftBack, bottomNormal, textureBottomLeft);
            //shapeVertices[20] = new VertexPositionNormalTexture(bottomRightBack, bottomNormal, textureBottomRight);
            //shapeVertices[21] = new VertexPositionNormalTexture(bottomLeftFront, bottomNormal, textureTopLeft);
            //shapeVertices[22] = new VertexPositionNormalTexture(bottomRightBack, bottomNormal, textureBottomRight);
            //shapeVertices[23] = new VertexPositionNormalTexture(bottomRightFront, bottomNormal, textureTopRight);

            //// Left face.
            //shapeVertices[24] = new VertexPositionNormalTexture(topLeftFront, leftNormal, textureTopRight);
            //shapeVertices[25] = new VertexPositionNormalTexture(bottomLeftBack, leftNormal, textureBottomLeft);
            //shapeVertices[26] = new VertexPositionNormalTexture(bottomLeftFront, leftNormal, textureBottomRight);
            //shapeVertices[27] = new VertexPositionNormalTexture(topLeftBack, leftNormal, textureTopLeft);
            //shapeVertices[28] = new VertexPositionNormalTexture(bottomLeftBack, leftNormal, textureBottomLeft);
            //shapeVertices[29] = new VertexPositionNormalTexture(topLeftFront, leftNormal, textureTopRight);

            //// Right face. 
            //shapeVertices[30] = new VertexPositionNormalTexture(topRightFront, rightNormal, textureTopLeft);
            //shapeVertices[31] = new VertexPositionNormalTexture(bottomRightFront, rightNormal, textureBottomLeft);
            //shapeVertices[32] = new VertexPositionNormalTexture(bottomRightBack, rightNormal, textureBottomRight);
            //shapeVertices[33] = new VertexPositionNormalTexture(topRightBack, rightNormal, textureTopRight);
            //shapeVertices[34] = new VertexPositionNormalTexture(topRightFront, rightNormal, textureTopLeft);
            //shapeVertices[35] = new VertexPositionNormalTexture(bottomRightBack, rightNormal, textureBottomRight);
        }

        public void RenderShape(GraphicsDevice device)
        {
            BuildShape();

            shapeBuffer = new VertexBuffer(device, typeof(VertexPositionNormalTexture), shapeVertices.Length, BufferUsage.None);
            shapeBuffer.SetData(shapeVertices);

            device.SetVertexBuffer(shapeBuffer);
            device.DrawPrimitives(PrimitiveType.TriangleList, 0, shapeTriangles);
        }
    }
}

We will use this basic shape to draw in the Augmented Reality View. In this view first we declare our necessary 3D World calculation variables.
private PhotoCamera _cam;                                   //Windows Phone Camera API object
        private Motion _motion;                                     //Motion  API object

        private GameTimer _gameTimer;                               //XNA Game Timer
        private DispatcherTimer _gestureTimer;                      //rewind timer after gesture to start motion api
        private int _gestureTime = 0;                               //seconds
        private const int _gestureMaxTime = 1;                      //seconds
        private SpriteBatch _spriteBatch;                           //XNA Sprite drawer
        private GraphicsDevice _device;                             //XNA DirectX Device object
        private UIElementRenderer BackgroundRenderer;

        private Matrix _cameraMatrix;                               //camera position and look at.
        private Matrix _projectionMatrix;                           //3d world projection matrix.
        private Matrix _phoneWorld = Matrix.CreateTranslation(0, 0, 0);   //phone position in real wold. (camera world)

        /// <summary>
        /// last yaw angle of windows phone device
        /// </summary>
        private float _yaw;
        /// <summary>
        /// last roll angle of windows phone device
        /// </summary>
        private float _roll;
        /// <summary>
        /// last pitch angle of windows phone device
        /// </summary>
        private float _pitch;

        /// <summary>
        /// Gesture States:
        /// no touch = OnMotion
        /// move your finger = OnFreeDrag
        /// on touch specific item in the screen = OnPressHold
        /// </summary>
        private enum GestureStateEnum { OnMotion, OnFreeDrag, OnPressHold }
        /// <summary>
        /// Current Gesture State on the camera view        
        /// </summary>
        private GestureStateEnum _gestureState = GestureStateEnum.OnMotion;

Now we could initialize Camera, XNA Engine, Motion detection and Gestures.

        private void InitializeCamera()
        {
            try
            {
                _cam = new Microsoft.Devices.PhotoCamera();
                viewfinderBrush.SetSource(_cam);
            }
            catch (Exception ex)
            {
                MessageBox.Show("There is a problem with camera.");
                NavigationService.Navigate(new Uri("/ARFinity;component/APMapView.xaml", UriKind.Relative));
            }
        }

        private void InitializeXNA()
        {
            _device = SharedGraphicsDeviceManager.Current.GraphicsDevice;

            // Set the sharing mode of the graphics device to turn on XNA rendering
            _device.SetSharingMode(true);

            // Create a new SpriteBatch, which can be used to draw textures.
            _spriteBatch = new SpriteBatch(_device);

            BackgroundRenderer = new UIElementRenderer(this, _device.Viewport.Width, _device.Viewport.Height);
            // Create a timer for this page
            _gameTimer = new GameTimer();
            _gameTimer.UpdateInterval = TimeSpan.FromTicks(333333);
            _gameTimer.Update += new EventHandler<GameTimerEventArgs>(_gameTimer_Update);
            _gameTimer.Draw += new EventHandler<GameTimerEventArgs>(_gameTimer_Draw);

            _cameraMatrix = Matrix.CreateLookAt(Vector3.Zero, Vector3.Forward, Vector3.Up);
            _projectionMatrix = Matrix.CreatePerspectiveFieldOfView(MathHelper.PiOver2, (float)(this.LayoutRoot.ActualWidth / this.LayoutRoot.ActualHeight), 0.01f, 10000.0f);

            foreach (LocationsVM.Location l in myVM.SelectedNearbyLocations)
            {
                l.Object3dEffect = new BasicEffect(_device);
                l.Object3dEffect.World = l.WorldMatrix;
                l.Object3dEffect.View = _cameraMatrix;
                l.Object3dEffect.Projection = _projectionMatrix;
                l.Object3dEffect.TextureEnabled = true;
            }

            _gameTimer.Start();
        }

        private void InitializeMotion()
        {
            if (!Motion.IsSupported)
            {
                MessageBox.Show("The Motion is not supported on this device.");
                NavigationService.Navigate(new Uri("/ARFinity;component/ARMapView.xaml", UriKind.Relative));
                //NavigationService.GoBack();
            }

            if (_motion == null)
            {
                try
                {
                    _motion = new Motion();
                    _motion.TimeBetweenUpdates = TimeSpan.FromMilliseconds(100);
                    _motion.CurrentValueChanged += new EventHandler<SensorReadingEventArgs<MotionReading>>(_motion_CurrentValueChanged);
                }
                catch (Exception ex)
                {
                    MessageBox.Show("There is a problem with motion detection.");
                    NavigationService.Navigate(new Uri("/ARFinity;component/ARMapView.xaml", UriKind.Relative));
                    //NavigationService.GoBack();
                }
            }

            try
            {
                _motion.Start();
            }
            catch (Exception ex)
            {
                MessageBox.Show("Unable to start the Motion API.");
                NavigationService.Navigate(new Uri("/ARFinity;component/ARMapView.xaml", UriKind.Relative));
                //NavigationService.GoBack();
            }
        }

        private void InitializeGestureTimer()
        {
            _gestureTimer = new DispatcherTimer();
            _gestureTimer.Tick += new EventHandler(_gestureTimer_Tick);
            _gestureTimer.Interval = TimeSpan.FromSeconds(1);
        }

        private void InitializeGestures()
        {
            TouchPanel.EnabledGestures = GestureType.FreeDrag | GestureType.Hold | GestureType.DragComplete | GestureType.Tap;
        }

After this kind of initializations now the matrix calculations and movements could be done. How to do it you could examine the source code. :)

Foursquare

To collect data we are using fousquare api. To do that we have included in this project a foursquare C# Client.

First of all we should have declared foursquare API v2 serialization objects.
using System.Collections.Generic;
using System.Runtime.Serialization;

namespace FoursquareApi.Objects
{
    [DataContract]
    public class Meta
    {
        [DataMember]
        public int code { get; set; }
        [DataMember]
        public string errorType { get; set; }
        [DataMember]
        public string errorDetail { get; set; }
        [DataMember]
        public Response response { get; set; }
    }

    [DataContract]
    public class Response
    {
        [DataMember]
        public List<Groups> groups { get; set; }
    }

    [DataContract]
    public class Groups
    {
        [DataMember]
        public string type { get; set; }
        [DataMember]
        public string name { get; set; }
        [DataMember]
        public List<Nearby> items { get; set; }
    }

    [DataContract]
    public class Nearby
    {
        [DataMember]
        public string id { get; set; }
        [DataMember]
        public string name { get; set; }
        [DataMember]
        public Contact contact { get; set; }
        [DataMember]
        public Location location { get; set; }
        [DataMember]
        public List<Category> categories { get; set; }
        [DataMember]
        public bool verified { get; set; }
        [DataMember]
        public Stat stats { get; set; }
        [DataMember]
        public HereNow hereNow { get; set; }
    }

    [DataContract]
    public class Contact
    {
        [DataMember]
        public string phone { get; set; }
        [DataMember]
        public string formattedPhone { get; set; }
    }

    [DataContract]
    public class Location
    {
        [DataMember]
        public double lat { get; set; }
        [DataMember]
        public double lng { get; set; }
        [DataMember]
        public int distance { get; set; }
        [DataMember]
        public string city { get; set; }
        [DataMember]
        public string address { get; set; }
    }

    [DataContract]
    public class Category
    {
        [DataMember]
        public string id { get; set; }
        [DataMember]
        public string name { get; set; }
        [DataMember]
        public string pluralName { get; set; }
        [DataMember]
        public string shortName { get; set; }
        [DataMember]
        public string icon { get; set; }
        [DataMember]
        public List<string> parents { get; set; }
        [DataMember]
        public bool primary { get; set; }
    }

    [DataContract]
    public class Stat
    {
        [DataMember]
        public int checkinsCount { get; set; }
        [DataMember]
        public int usersCount { get; set; }
        [DataMember]
        public int tipCount { get; set; }
    }

    [DataContract]
    public class HereNow
    {
        [DataMember]
        public int count { get; set; }
    }
}

After that we could write the engine
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Runtime.Serialization.Json;
using System.Text;
using System.Windows;

using FoursquareApi.Objects;

namespace FoursquareApi
{
    public class FoursquareClient
    {
        /// <summary>
        /// gets the foursquare application client id
        /// </summary>
        public string ClientId { get; private set; }
        /// <summary>
        /// gets the foursquare application client secret 
        /// </summary>
        public string ClientSecret { get; private set; }

        public delegate void GetNearbyVenuesCompletedEventHandler(List<Nearby> nearbys);
        public event GetNearbyVenuesCompletedEventHandler GetNearbyVenuesCompleted;

        public event EventHandler GetNearbyVenuesFailed;

        /// <summary>
        /// constructor of Foursquare application client 
        /// </summary>
        /// <param name="_clientId">foursquare application client id</param>
        /// <param name="_clientSecret">foursquare application client secret</param>
        public FoursquareClient(string _clientId, string _clientSecret)
        {
            ClientId = _clientId;
            ClientSecret = _clientSecret;
        }

        /// <summary>
        /// Gets the nearby locations from foursquare api asyncrously
        /// </summary>
        /// <param name="lat">Latitude</param>
        /// <param name="lng">Longtitude</param>
        /// <param name="radius">Search Radius in meters</param>
        public void GetNearbyVenuesAsync(double lat, double lng, int radius)
        {
            StringBuilder urlBuilder = new StringBuilder("https://api.foursquare.com/v2/venues/search?");
            urlBuilder.Append("ll=" + lat.ToString().Replace(",", ".") + "," + lng.ToString().Replace(",", "."));
            urlBuilder.Append(@"&radius=" + radius.ToString());
            urlBuilder.Append("&client_id=" + ClientId);
            urlBuilder.Append("&client_secret=" + ClientSecret);

            HttpWebRequest webRequest = (HttpWebRequest)HttpWebRequest.Create(urlBuilder.ToString());
            webRequest.BeginGetResponse(new AsyncCallback(FinishGetNearbyRequest), webRequest);
        }

        private void FinishGetNearbyRequest(IAsyncResult result)
        {
            string content = string.Empty;
            //List<Nearby> nearbys = new List<Nearby>();
            Meta m = new Meta();

            try
            {
                HttpWebResponse response = (result.AsyncState as HttpWebRequest).EndGetResponse(result) as HttpWebResponse;
                using (StreamReader sr = new StreamReader(response.GetResponseStream()))
                {
                    content = sr.ReadToEnd();
                }

                using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(content)))
                {
                    DataContractJsonSerializer ser = new DataContractJsonSerializer(m.GetType());
                    m = ser.ReadObject(ms) as Meta;
                }

                Deployment.Current.Dispatcher.BeginInvoke(delegate()
                {
                    if (GetNearbyVenuesCompleted != null)
                        GetNearbyVenuesCompleted(m.response.groups[0].items);
                });
            }
            catch (Exception ex)
            {
                Deployment.Current.Dispatcher.BeginInvoke(delegate()
                {
                    if (GetNearbyVenuesFailed != null)
                        GetNearbyVenuesFailed(this, null);
                });
            }
        }
    }
}

follow me on twitter: twitter.com/oguzkoroglu

my linked in profile : http://tr.linkedin.com/in/oguzkoroglu/

my personal web site : http://oguzkoroglu.net/

Last edited Jun 19, 2014 at 6:58 AM by ogu, version 4