Okay, I think I figured out a workaround.
I'm leaving the Margin property untouched throughout the entire process. This avoids the original problem but, as shown before, makes the menu overlap the button (see second animation in my original question).
Now I have to compensate for the unwanted offset, but I can't use the VerticalOffset property (because removing it after the animation has finished will again result in sporadic flicker). Instead, I use a clipping geometry and animate it so that it always clips the part of the menu which overlaps the button:
private void ContextMenu_Opened(object sender, RoutedEventArgs e)
{
/* Prepare translation animation as shown before... */
RectangleGeometry clipGeometry = new RectangleGeometry(new Rect(
new Point(-contextMenu.Margin.Left, 0),
new Size(contextMenu.ActualWidth + contextMenu.Margin.Left + contextMenu.Margin.Right,
contextMenu.ActualHeight + contextMenu.Margin.Bottom)));
contextMenu.RegisterName("RectangleGeometry", clipGeometry);
contextMenu.Clip = clipGeometry;
RectAnimation clippingAnimation = new RectAnimation()
{
Duration = new Duration(new TimeSpan(0, 0, 0, 0, 200)),
EasingFunction = new PowerEase() { EasingMode = EasingMode.EaseOut, Power = 3 },
From = new Rect(
new Point(-contextMenu.Margin.Left, 20),
new Size(contextMenu.ActualWidth + contextMenu.Margin.Left + contextMenu.Margin.Right,
contextMenu.ActualHeight - 20 + contextMenu.Margin.Bottom))
};
SetTargetProperty(clippingAnimation, new PropertyPath(RectangleGeometry.RectProperty));
SetTargetName(clippingAnimation, "RectangleGeometry");
storyboard.Children.Add(clippingAnimation);
storyboard.Completed += new EventHandler((animation, eventArgs) =>
{
contextMenu.RenderTransform = null;
contextMenu.Clip = null;
});
storyboard.Begin(this);
}