Tech Off Post

Single Post Permalink

View Thread: Mouse Up, Mouse Down through hoverButton of kinect
  • User profile image
    Beginner91

    Hello Everyone,

    I want implement drag and drop through hoverButton of kinect.

    public partial class MainMenu : UserControl
        {
            private KinectSensor _Kinect;
            private WriteableBitmap _ColorImageBitmap;
            private Int32Rect _ColorImageBitmapRect;
            private int _ColorImageStride;
            private Skeleton[] FrameSkeletons;
    
            List<Button> buttons;
            static Button selected;
    
            float handX;
            float handY;
    
            
            
    
    
            public MainMenu()
            {
    
                InitializeComponent();
                InitializeButtons();
    
                kinectButton.Click += new RoutedEventHandler(kinectButton_Click);
    
                this.Loaded += (s, e) => { DiscoverKinectSensor(); };
                this.Unloaded += (s, e) => { this.Kinect = null; };
    
            }
    
            private void Number3Operator()
            {
    
                Random random3 = new Random();
                int randomNumber3 = random3.Next(0, 3);
    
            }
    
            #region "Hand Gesture"
    
            //initialize buttons to be checked
            private void InitializeButtons()
            {
                buttons = new List<Button> { GAME1, GAME2, QUIT };
            }
            //raise event for Kinect sensor status changed
            private void DiscoverKinectSensor()
            {
                KinectSensor.KinectSensors.StatusChanged += KinectSensors_StatusChanged;
                this.Kinect = KinectSensor.KinectSensors.FirstOrDefault(x => x.Status == KinectStatus.Connected);
            }
    
    
            private void KinectSensors_StatusChanged(object sender, StatusChangedEventArgs e)
            {
                switch (e.Status)
                {
                    case KinectStatus.Connected:
                        if (this.Kinect == null)
                        {
                            this.Kinect = e.Sensor;
                        }
                        break;
                    case KinectStatus.Disconnected:
                        if (this.Kinect == e.Sensor)
                        {
                            this.Kinect = null;
                            this.Kinect = KinectSensor.KinectSensors.FirstOrDefault(x => x.Status == KinectStatus.Connected);
                            if (this.Kinect == null)
                            {
                                MessageBox.Show("Sensor Disconnected. Please reconnect to continue.");
                            }
                        }
                        break;
                }
            }
    
            public KinectSensor Kinect
            {
                get { return this._Kinect; }
                set
                {
                    if (this._Kinect != value)
                    {
                        if (this._Kinect != null)
                        {
                            UninitializeKinectSensor(this._Kinect);
                            this._Kinect = null;
                        }
                        if (value != null && value.Status == KinectStatus.Connected)
                        {
                            this._Kinect = value;
                            InitializeKinectSensor(this._Kinect);
                        }
                    }
                }
            }
    
            private void UninitializeKinectSensor(KinectSensor kinectSensor)
            {
                if (kinectSensor != null)
                {
                    kinectSensor.Stop();
                    kinectSensor.ColorFrameReady -= Kinect_ColorFrameReady;
                    kinectSensor.SkeletonFrameReady -= Kinect_SkeletonFrameReady;
                }
            }
    
            private void InitializeKinectSensor(KinectSensor kinectSensor)
            {
                if (kinectSensor != null)
                {
                    ColorImageStream colorStream = kinectSensor.ColorStream;
                    colorStream.Enable();
                    this._ColorImageBitmap = new WriteableBitmap(colorStream.FrameWidth, colorStream.FrameHeight,
                        96, 96, PixelFormats.Bgr32, null);
                    this._ColorImageBitmapRect = new Int32Rect(0, 0, colorStream.FrameWidth, colorStream.FrameHeight);
                    this._ColorImageStride = colorStream.FrameWidth * colorStream.FrameBytesPerPixel;
                    videoStream.Source = this._ColorImageBitmap;
    
                    kinectSensor.SkeletonStream.Enable(new TransformSmoothParameters()
                    {
                        Correction = 0.5f,
                        JitterRadius = 0.05f,
                        MaxDeviationRadius = 0.04f,
                        Smoothing = 0.5f
                    });
    
                    kinectSensor.SkeletonFrameReady += Kinect_SkeletonFrameReady;
                    kinectSensor.ColorFrameReady += Kinect_ColorFrameReady;
                    kinectSensor.Start();
                    this.FrameSkeletons = new Skeleton[this.Kinect.SkeletonStream.FrameSkeletonArrayLength];
    
                }
            }
    
            private void Kinect_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
            {
                using (ColorImageFrame frame = e.OpenColorImageFrame())
                {
                    if (frame != null)
                    {
                        byte[] pixelData = new byte[frame.PixelDataLength];
                        frame.CopyPixelDataTo(pixelData);
                        this._ColorImageBitmap.WritePixels(this._ColorImageBitmapRect, pixelData,
                            this._ColorImageStride, 0);
                    }
                }
            }
    
            private void Kinect_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
            {
                using (SkeletonFrame frame = e.OpenSkeletonFrame())
                {
                    if (frame != null)
                    {
                        frame.CopySkeletonDataTo(this.FrameSkeletons);
                        Skeleton skeleton = GetPrimarySkeleton(this.FrameSkeletons);
    
                        if (skeleton == null)
                        {
                            kinectButton.Visibility = Visibility.Collapsed;
                        }
                        else
                        {
                            Joint primaryHand = GetPrimaryHand(skeleton);
                            TrackHand(primaryHand);
    
                        }
                    }
                }
            }
    
            //track and display hand
            private void TrackHand(Joint hand)
            {
                if (hand.TrackingState == JointTrackingState.NotTracked)
                {
                    kinectButton.Visibility = System.Windows.Visibility.Collapsed;
                }
                else
                {
                    kinectButton.Visibility = System.Windows.Visibility.Visible;
                    
                    //DepthImagePoint point = this.Kinect.MapSkeletonPointToDepth(hand.Position, DepthImageFormat.Resolution640x480Fps30);
                    DepthImagePoint point = this.Kinect.CoordinateMapper.MapSkeletonPointToDepthPoint(hand.Position, DepthImageFormat.Resolution640x480Fps30);
                    handX = (int)((point.X * LayoutRoot.ActualWidth / this.Kinect.DepthStream.FrameWidth) -
                        (kinectButton.ActualWidth / 2.0));
                    handY = (int)((point.Y * LayoutRoot.ActualHeight / this.Kinect.DepthStream.FrameHeight) -
                        (kinectButton.ActualHeight / 2.0));
                    Canvas.SetLeft(kinectButton, handX);
                    Canvas.SetTop(kinectButton, handY);
    
                    if (isHandOver(kinectButton, buttons)) kinectButton.Hovering();
                    else kinectButton.Release();
                    if (hand.JointType == JointType.HandRight)
                    {
                        kinectButton.ImageSource = "/Images/RightHand.png";
                        kinectButton.ActiveImageSource = "/Images/RightHand.png";
                    }
                    else
                    {
                        kinectButton.ImageSource = "/Images/LeftHand.png";
                        kinectButton.ActiveImageSource = "/Images/LeftHand.png";
                    }
                }
            }
    
            //detect if hand is overlapping over any button
            private bool isHandOver(FrameworkElement hand, List<Button> buttonslist)
            {
                var handTopLeft = new Point(Canvas.GetLeft(hand), Canvas.GetTop(hand));
                var handX = handTopLeft.X + hand.ActualWidth / 2;
                var handY = handTopLeft.Y + hand.ActualHeight / 2;
    
                foreach (Button target in buttonslist)
                {
                    Point targetTopLeft = new Point(Canvas.GetLeft(target), Canvas.GetTop(target));
                    if (handX > targetTopLeft.X &&
                        handX < targetTopLeft.X + target.Width &&
                        handY > targetTopLeft.Y &&
                        handY < targetTopLeft.Y + target.Height)
                    {
                        selected = target;
                        return true;
                    }
                }
                return false;
            }
    
            //get the hand closest to the Kinect sensor
            private static Joint GetPrimaryHand(Skeleton skeleton)
            {
                Joint primaryHand = new Joint();
                if (skeleton != null)
                {
                    primaryHand = skeleton.Joints[JointType.HandLeft];
                    Joint rightHand = skeleton.Joints[JointType.HandRight];
                    if (rightHand.TrackingState != JointTrackingState.NotTracked)
                    {
                        if (primaryHand.TrackingState == JointTrackingState.NotTracked)
                        {
                            primaryHand = rightHand;
                        }
                        else
                        {
                            if (primaryHand.Position.Z > rightHand.Position.Z)
                            {
                                primaryHand = rightHand;
                            }
                        }
                    }
                }
                return primaryHand;
            }
    
            //get the skeleton closest to the Kinect sensor
            private static Skeleton GetPrimarySkeleton(Skeleton[] skeletons)
            {
                Skeleton skeleton = null;
                if (skeletons != null)
                {
                    for (int i = 0; i < skeletons.Length; i++)
                    {
                        if (skeletons[i].TrackingState == SkeletonTrackingState.Tracked)
                        {
                            if (skeleton == null)
                            {
                                skeleton = skeletons[i];
                            }
                            else
                            {
                                if (skeleton.Position.Z > skeletons[i].Position.Z)
                                {
                                    skeleton = skeletons[i];
                                }
                            }
                        }
                    }
                }
                return skeleton;
            }
    
            void kinectButton_Click(object sender, RoutedEventArgs e)
            {
                selected.RaiseEvent(new RoutedEventArgs(Button.ClickEvent, selected));
            }

    So far, I've done only click event, but I want to do like drag and drop, you click on a button and drag it, after that you put that button in another label or frame, holding it for 5 seconds, after that you drop it there. How can I do this, can you give me advice?