Workspace/Assets/Modules/ShrinkTutorial/Runtime/UI/ShrinkTutorialMask.cs

using UnityEngine;
using UnityEngine.UI;

namespace ShrinkTutorial
{
    public class ShrinkTutorialMask : MaskableGraphic, ICanvasRaycastFilter
    {
        private Rect _holeRect;
        private ShrinkTutorialMaskShape _shape;
        private bool _hasHole;

        public void SetHighlight(Rect screenRect, ShrinkTutorialMaskShape shape, float padding)
        {
            enabled = true;
            raycastTarget = true;

            var camera = GetEventCamera();
            var minScreen = new Vector2(screenRect.xMin - padding, screenRect.yMin - padding);
            var maxScreen = new Vector2(screenRect.xMax + padding, screenRect.yMax + padding);

            RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, minScreen, camera, out var minLocal);
            RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, maxScreen, camera, out var maxLocal);

            _holeRect = Rect.MinMaxRect(
                Mathf.Min(minLocal.x, maxLocal.x),
                Mathf.Min(minLocal.y, maxLocal.y),
                Mathf.Max(minLocal.x, maxLocal.x),
                Mathf.Max(minLocal.y, maxLocal.y));
            _shape = shape;
            _hasHole = shape != ShrinkTutorialMaskShape.None;
            SetVerticesDirty();
        }

        public void ClearHighlight()
        {
            _hasHole = false;
            _shape = ShrinkTutorialMaskShape.None;
            _holeRect = default;
            raycastTarget = false;
            SetVerticesDirty();
            enabled = false;
        }

        public bool IsRaycastLocationValid(Vector2 screenPoint, Camera eventCamera)
        {
            if (!_hasHole)
                return false;

            RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, screenPoint, eventCamera, out var localPoint);
            if (_shape == ShrinkTutorialMaskShape.Circle)
            {
                var radius = Mathf.Min(_holeRect.width, _holeRect.height) * 0.5f;
                return Vector2.SqrMagnitude(localPoint - _holeRect.center) > radius * radius;
            }

            return !_holeRect.Contains(localPoint);
        }

        protected override void OnPopulateMesh(VertexHelper vh)
        {
            vh.Clear();

            var outer = rectTransform.rect;
            if (!_hasHole)
                return;

            var hole = ClampRect(_holeRect, outer);

            if (hole.yMax < outer.yMax)
            {
                AddQuad(vh,
                    new Vector2(outer.xMin, hole.yMax),
                    new Vector2(outer.xMin, outer.yMax),
                    new Vector2(outer.xMax, outer.yMax),
                    new Vector2(outer.xMax, hole.yMax));
            }

            if (hole.yMin > outer.yMin)
            {
                AddQuad(vh,
                    new Vector2(outer.xMin, outer.yMin),
                    new Vector2(outer.xMin, hole.yMin),
                    new Vector2(outer.xMax, hole.yMin),
                    new Vector2(outer.xMax, outer.yMin));
            }

            if (hole.xMin > outer.xMin)
            {
                AddQuad(vh,
                    new Vector2(outer.xMin, hole.yMin),
                    new Vector2(outer.xMin, hole.yMax),
                    new Vector2(hole.xMin, hole.yMax),
                    new Vector2(hole.xMin, hole.yMin));
            }

            if (hole.xMax < outer.xMax)
            {
                AddQuad(vh,
                    new Vector2(hole.xMax, hole.yMin),
                    new Vector2(hole.xMax, hole.yMax),
                    new Vector2(outer.xMax, hole.yMax),
                    new Vector2(outer.xMax, hole.yMin));
            }
        }

        private static Rect ClampRect(Rect source, Rect bounds)
        {
            return Rect.MinMaxRect(
                Mathf.Clamp(source.xMin, bounds.xMin, bounds.xMax),
                Mathf.Clamp(source.yMin, bounds.yMin, bounds.yMax),
                Mathf.Clamp(source.xMax, bounds.xMin, bounds.xMax),
                Mathf.Clamp(source.yMax, bounds.yMin, bounds.yMax));
        }

        private void AddQuad(VertexHelper vh, Vector2 bottomLeft, Vector2 topLeft, Vector2 topRight, Vector2 bottomRight)
        {
            var startIndex = vh.currentVertCount;
            vh.AddVert(bottomLeft, color, Vector2.zero);
            vh.AddVert(topLeft, color, Vector2.up);
            vh.AddVert(topRight, color, Vector2.one);
            vh.AddVert(bottomRight, color, Vector2.right);
            vh.AddTriangle(startIndex, startIndex + 1, startIndex + 2);
            vh.AddTriangle(startIndex, startIndex + 2, startIndex + 3);
        }

        private Camera GetEventCamera()
        {
            var currentCanvas = canvas ? canvas : GetComponentInParent<Canvas>();
            if (!currentCanvas || currentCanvas.renderMode == RenderMode.ScreenSpaceOverlay)
                return null;

            return currentCanvas.worldCamera;
        }
    }
}