Automation With Windows UI Automation

UIA shipped with .Net 3.0 did not receive the attention it really deserves . UIA can be used to automate native windows applications , WPF applications as well as Silverlight applications . There are two things one must consider when automating

1. Reference to the Automation Element
2. The user pattern to be invoked 

That's all . The AutomationElement.RootElement represents the desktop , all other UI elements are children of the desktop . We need to search the tree to find the required automation element . The tree hierarchy can be found by a tool called UISpy that comes as a part of Windows SDK . Then we need to determine what pattern is  to be used , eg  combobox uses  ExpandCollapse Pattern , textbox uses TextPattern and  button uses  Invoke pattern . (MSDN has all information )

Lets see how its done using 2 simple automation actions  

Action  1 : Opening notepad and  putting some junk text in it 

using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Diagnostics;using System.Windows.Automation;using System.Windows.Automation.Provider;using System.Windows.Automation.Text;using System.Windows.Forms;namespace AutomateNotepad{ class Program { static void Main(string[] args) { // Make sure that all notepad process are killed Process[] notepadProcceses = Process.GetProcessesByName("notepad"); foreach (Process notepadProc in notepadProcceses) { notepadProc.Kill(); } // Start a notepad window Process notepadProcces = Process.Start("notepad"); // Get the reference to the notepad window AutomationElement notepadWindow = AutomationElement.RootElement.FindFirst(TreeScope.Descendants , new PropertyCondition(AutomationElement.ClassNameProperty,"Notepad")) ; // Get the reference to the notepad text box AutomationElement notepadTextBox = AutomationElement.RootElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.ClassNameProperty, "Edit")); TextPattern putText = notepadTextBox.GetCurrentPattern(TextPattern.Pattern) as TextPattern; putText.DocumentRange.Select(); SendKeys.SendWait(" You there ! why are you looking at me !"); Console.ReadKey(); } }}

Action 2 : Walk through all the elements in a Silverlight page

The  XAML file of the Silverlight page is  as follows

 

<UserControl x:Class="SilverlightTestPage.Page" xmlns="https://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="https://schemas.microsoft.com/winfx/2006/xaml" Width="400" Height="300" xmlns:System="clr-namespace:System;assembly=mscorlib"> <Grid x:Name="LayoutRoot" Background="White"> <Rectangle Height="51" VerticalAlignment="Top" Fill="#FF0F36E2" Stroke="#FF000000"/> <Rectangle Height="2" HorizontalAlignment="Right" Margin="0,0,8,0" VerticalAlignment="Bottom" Width="0" Fill="#FF0F36E2" Stroke="#FF000000"/> <TextBlock Height="32" Margin="53,8,126,0" VerticalAlignment="Top" Text="Automation Demo" TextWrapping="Wrap"/> <Grid HorizontalAlignment="Left" Margin="0,51,0,0" Width="80" Background="#FF904F4F"> <Button x:Name="myButton1" Background="#FF339529" Height="40" Margin="8,27,8,0" VerticalAlignment="Top" Content="Click Me " Click="myButton1_Click"> <Button.BorderBrush> <LinearGradientBrush EndPoint="0.5,1" StartPoint="0.5,0"> <GradientStop Color="#FFA3AEB9"/> <GradientStop Color="#FF8399A9" Offset="0.375"/> <GradientStop Color="#FF718597" Offset="0.375"/> <GradientStop Color="#FF4DACF3" Offset="1"/> </LinearGradientBrush> </Button.BorderBrush> </Button> </Grid> <TextBlock HorizontalAlignment="Left" Margin="0,134,0,147" Width="80" Text="ComboBox1" TextWrapping="Wrap"/> <ComboBox x:Name="myCombo1" Height="19" HorizontalAlignment="Left" Margin="0,0,0,114" VerticalAlignment="Bottom" Width="72"> <System:String>String1</System:String> <System:String>String2</System:String> </ComboBox> <CheckBox x:Name="myCheck1" Height="19" HorizontalAlignment="Left" Margin="8,0,0,61" VerticalAlignment="Bottom" Width="20" Content="CheckBox"/> <TextBlock Height="26" HorizontalAlignment="Left" Margin="0,0,0,84" VerticalAlignment="Bottom" Width="72" Text="Checkbox1" TextWrapping="Wrap"/> <RadioButton x:Name="myRadio1" Height="15" HorizontalAlignment="Left" Margin="112,91,0,0" VerticalAlignment="Top" Width="17" Content="RadioButton"/> <TextBlock Height="21" HorizontalAlignment="Left" Margin="93,66,0,0" VerticalAlignment="Top" Width="84" Text="radiobutton1" TextWrapping="Wrap"/> <TextBox x:Name="myTextBox1" Margin="102,134,126,132" Text="Put Some Text Here" TextWrapping="Wrap"/> <TextBlock x:Name="myTextBlock1" Height="45" Margin="112,0,51,39" VerticalAlignment="Bottom" Text="TextBlock" TextWrapping="Wrap"/> </Grid></UserControl>

The code for automating it is as follows

using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Diagnostics;using System.Windows.Automation;using System.Windows.Automation.Provider;using System.Windows.Automation.Text;using System.Windows.Forms;using System.Threading;namespace WindowsUIAutomationDemo{ class Program { static void Main(string[] args) { // Start the internet explorer process Process p = Process.Start("iexplore"); Thread.Sleep(10000); // Get the handle to the ieWindow AutomationElement ieWindow = AutomationElement.RootElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.ClassNameProperty,"IEFrame")); Thread.Sleep(5000); if (ieWindow == null) { Console.WriteLine("Unable to get handle to the ie main window!"); } else { Thread.Sleep(5000); // Specify the condition for getting a reference to the edit control Condition[] condition = new Condition[3]; condition[0] = new PropertyCondition(AutomationElement.ClassNameProperty, "Edit"); condition[1] = new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Edit); condition[2] = new PropertyCondition(AutomationElement.NameProperty, "Address"); // Getting a reference to the iebar AutomationElement iebar = ieWindow.FindFirst(TreeScope.Descendants, new AndCondition(condition)); if (iebar == null) { Console.WriteLine("Unable to get handle to the window !"); } else { // Set focus and go to the address iebar.SetFocus(); SendKeys.SendWait(@"E:\WindowsUIAutomationDemo\SilverlightTestPage\SilverlightTestPage\Bin\Debug\TestPage.html"); SendKeys.SendWait("{Enter}"); Thread.Sleep(20000); // Get the reference to the silverlight control AutomationElement silverlightElement = ieWindow.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.ClassNameProperty, "MicrosoftSilverlight")); if (silverlightElement == null) { Console.WriteLine("Unable to get access to silverligt contorl"); } else { // click the click me button AutomationElement clickButton = silverlightElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.AutomationIdProperty, "myButton1")); InvokePattern invokePattern = clickButton.GetCurrentPattern(InvokePattern.Pattern) as InvokePattern; invokePattern.Invoke(); Thread.Sleep(2000); // Expand combo box AutomationElement comboBox = silverlightElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.AutomationIdProperty, "myCombo1")); ExpandCollapsePattern expandCollapsePattern = comboBox.GetCurrentPattern(ExpandCollapsePattern.Pattern) as ExpandCollapsePattern; expandCollapsePattern.Expand(); Thread.Sleep(2000) ; // Selct the first item AutomationElement firstItemincombo = silverlightElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.NameProperty, "String1")); SelectionItemPattern itemSelection = firstItemincombo.GetCurrentPattern(SelectionItemPattern.Pattern) as SelectionItemPattern; itemSelection.Select(); Thread.Sleep(1000) ; expandCollapsePattern.Collapse(); //Get the radio checkbox AutomationElement checkboxPattern = silverlightElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.AutomationIdProperty, "myCheck1")); TogglePattern tooglePattern = checkboxPattern.GetCurrentPattern(TogglePattern.Pattern) as TogglePattern; tooglePattern.Toggle(); Thread.Sleep(1000) ; // Get the radio button AutomationElement radioButton = silverlightElement.FindFirst(TreeScope.Descendants,new PropertyCondition(AutomationElement.AutomationIdProperty,"myRadio1")); SelectionItemPattern radioCheck =radioButton.GetCurrentPattern(SelectionItemPattern.Pattern) as SelectionItemPattern; radioCheck.Select(); Thread.Sleep(1000) ; // Get the textbox and put something there AutomationElement textBox = silverlightElement.FindFirst(TreeScope.Descendants,new PropertyCondition(AutomationElement.AutomationIdProperty,"myTextBox1")); ValuePattern valuePattern = textBox.GetCurrentPattern(ValuePattern.Pattern) as ValuePattern; valuePattern.SetValue("Yo yo , i am entering text !"); Thread.Sleep(1000); } //AutomationElement silverlightControl = ieWindow.FindFirst(TreeScope.Descendants,new PropertyCondition( } } Console.ReadKey(); } }}