source: OniSplit/Akira/PolygonUtils.cs

using System;
using System.Collections.Generic;

namespace Oni.Akira
{
    internal class PolygonUtils
    {
        public static List<Vector3> ClipToPlane(List<Vector3> points, Plane plane)
        {
            var signs = new int[points.Count];
            var negativeCount = 0;
            var positiveCount = 0;
            var start = 0;

            for (int i = 0; i < points.Count; i++)
            {
                signs[i] = RelativePosition(points[i], plane);

                if (signs[i] >= 0)
                    positiveCount++;

                if (signs[i] <= 0)
                    negativeCount++;
            }

            if (negativeCount == points.Count)
                return null;

            if (positiveCount == points.Count)
                return points;

            var newPoints = new List<Vector3>();

            for (int i = 0; i < points.Count; i++)
            {
                int i0 = (i + start) % points.Count;
                int i1 = (i + start + 1) % points.Count;

                int s0 = signs[i0];
                int s1 = signs[i1];

                if (s0 >= 0)
                {
                    newPoints.Add(points[i0]);

                    if (s0 > 0 && s1 < 0)
                        newPoints.Add(Intersect(points[i0], points[i1], plane));
                }
                else
                {
                    if (s0 < 0 && s1 > 0)
                        newPoints.Add(Intersect(points[i1], points[i0], plane));
                }
            }

            return newPoints;
        }

        private static Vector3 Intersect(Vector3 p0, Vector3 p1, Plane plane)
        {
            Vector3 dir = p1 - p0;

            float dND = plane.DotNormal(dir);

            if (Math.Abs(dND) < 1e-05f)
                throw new InvalidOperationException();

            float distance = (-plane.D - plane.DotNormal(p0)) / dND;

            if (distance < 0.0f)
            {
                if (distance < -1e-05f)
                    throw new InvalidOperationException();

                return p0;
            }
            else
            {
                return p0 + dir * distance;
            }
        }

        private static int RelativePosition(Vector3 point, Plane plane)
        {
            float pos = plane.DotCoordinate(point);

            if (pos < -1e-05f)
                return -1;
            else if (pos > 1e-05f)
                return 1;
            else
                return 0;
        }
    }
}