<Window x:Class="ScaleScrolling.MainWindow"
        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
        xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
        xmlns:local="clr-namespace:ScaleScrolling"
        mc:Ignorable="d"
        Title="MainWindow" Height="450" Width="800">
    <Grid>

        <local:ScrollViewerEx HorizontalScrollBarVisibility="Auto" VerticalScrollBarVisibility="Auto" CanContentScroll="True"
                      MouseWheel="ScrollViewer_PreviewMouseWheel"
                      PreviewMouseDown="ScrollViewerEx_PreviewMouseDown"
                      PreviewMouseUp="ScrollViewerEx_PreviewMouseUp"
                      PreviewMouseMove="ScrollViewerEx_PreviewMouseMove">
                <Canvas PreviewMouseWheel="Canvas_PreviewMouseWheel" Width="300" Height="300">
                    <Canvas.LayoutTransform>
                        <ScaleTransform x:Name="PART_ScaleTransform"/>
                    </Canvas.LayoutTransform>
                    <Rectangle Canvas.Left="20" Canvas.Top="20" Fill="Red" Stroke="Black" Width="200" Height="200"/>
                    <Rectangle Canvas.Left="50" Canvas.Top="50" Fill="Green" Stroke="Black" Width="200" Height="200"/>
                    <Rectangle Canvas.Left="80" Canvas.Top="80" Fill="Blue" Stroke="Black" Width="200" Height="200"/>
                </Canvas>
        </local:ScrollViewerEx>
    </Grid>
</Window>

using System.Windows;
using System.Windows.Controls;
using System.Windows.Input;

namespace ScaleScrolling
{
    /// <summary>
    /// Interaction logic for MainWindow.xaml
    /// </summary>
    public partial class MainWindow : Window
    {
        public MainWindow()
        {
            InitializeComponent();
        }

        private void Canvas_PreviewMouseWheel(object sender, MouseWheelEventArgs e)
        {
            var mousePosition = e.GetPosition((IInputElement)sender);
            PART_ScaleTransform.CenterX = mousePosition.X;
            PART_ScaleTransform.CenterY = mousePosition.Y;
            if (e.Delta > 0)
            {
                PART_ScaleTransform.ScaleX += 0.1;
                PART_ScaleTransform.ScaleY += 0.1;
            }
            else
            {
                PART_ScaleTransform.ScaleX -= 0.1;
                PART_ScaleTransform.ScaleY -= 0.1;
            }
        }

        private void ScrollViewer_PreviewMouseWheel(object sender, MouseWheelEventArgs e)
        {
            e.Handled = true;
        }

        private Point _startMovingPoint;
        private double _startHorizontalOffset;
        private double _startVerticalOffset;
        private bool _startMoving;
        private void ScrollViewerEx_PreviewMouseDown(object sender, MouseButtonEventArgs e)
        {
            var scrollViewer = (ScrollViewer)sender;
            if (e.LeftButton == MouseButtonState.Pressed)
            {
                scrollViewer.CaptureMouse();
                _startMovingPoint = e.GetPosition((IInputElement)sender);
                _startHorizontalOffset = scrollViewer.HorizontalOffset;
                _startVerticalOffset = scrollViewer.VerticalOffset;
                _startMoving = true;

            }
        }

        private void ScrollViewerEx_PreviewMouseUp(object sender, MouseButtonEventArgs e)
        {
            var scrollViewer = (ScrollViewer)sender;
            if (scrollViewer.IsMouseCaptured)
            {
                scrollViewer.ReleaseMouseCapture();
                _startMoving = false;
            }
        }

        private void ScrollViewerEx_PreviewMouseMove(object sender, MouseEventArgs e)
        {
            var scrollViewer = (ScrollViewer)sender;
            if (_startMoving && e.LeftButton == MouseButtonState.Pressed)
            {
                var point = e.GetPosition((IInputElement)sender);
                var delta = point - _startMovingPoint;
                scrollViewer.ScrollToHorizontalOffset(_startHorizontalOffset - delta.X);
                scrollViewer.ScrollToVerticalOffset(_startVerticalOffset - delta.Y);
            }
        }
    }
}

using System.Windows.Controls;
using System.Windows.Input;

namespace ScaleScrolling
{
    public class ScrollViewerEx : ScrollViewer
    {
        protected override void OnMouseWheel(MouseWheelEventArgs e) { }
    }
}