From 94bed91d064da144519aa264869d3c415a4fd99e Mon Sep 17 00:00:00 2001 From: Miles Gratz Date: Tue, 11 Apr 2017 07:37:11 -0500 Subject: [PATCH] Initial commit --- FastLookup.psm1 | 55 ++++++++++++++ Get-FastLookup.ps1 | 83 +++++++++++++++++++++ New-FastLookup.ps1 | 121 +++++++++++++++++++++++++++++++ README.md | 99 ++++++++++++++++++++++++++ Test-FastLookup.ps1 | 170 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 528 insertions(+) create mode 100644 FastLookup.psm1 create mode 100644 Get-FastLookup.ps1 create mode 100644 New-FastLookup.ps1 create mode 100644 Test-FastLookup.ps1 diff --git a/FastLookup.psm1 b/FastLookup.psm1 new file mode 100644 index 0000000..50b3e0c --- /dev/null +++ b/FastLookup.psm1 @@ -0,0 +1,55 @@ +#requires -version 3 +<# +.SYNOPSIS + Lookup a value in an array faster than Where-Object + +.DESCRIPTION + Improve the speed of looking up a value in an array by creating a hashtable index. + Good for looking up results in very large arrays or CSV files (e.g Import-Csv) + +.NOTES + Version: 1.0 + Author: Miles Gratz + Creation Date: April 10, 2017 + Purpose/Change: Initial script development + +.EXAMPLE + + $array = 1..10000000 + $hashtable = New-FastLookup $array + + Measure-Command { $array | Where-Object { $_ -eq 199999 } } + + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 9 + Milliseconds : 714 + + Measure-Command { Get-FastLookup -Value 199999 -Array $array -Table $hashtable } + + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 0 + Milliseconds : 65 + + [NOTE] Performance test on Windows 10 x64 (i5-6200U, 8GB RAM, SSD) +#> + +$Functions = @( Get-ChildItem -Path $PSScriptRoot\*.ps1 -ErrorAction SilentlyContinue ) + +foreach ($Function in $Functions) +{ + Try + { + . $Function.FullName + } + Catch + { + Write-Error -Message "Failed to import function $($Function.FullName): $_" + } +} + +Export-ModuleMember -Alias 'FastLookup' +Export-ModuleMember -Function $Functions.BaseName \ No newline at end of file diff --git a/Get-FastLookup.ps1 b/Get-FastLookup.ps1 new file mode 100644 index 0000000..3b33b39 --- /dev/null +++ b/Get-FastLookup.ps1 @@ -0,0 +1,83 @@ +function Get-FastLookup { + <# + .SYNOPSIS + Lookup a value in an array faster than Where-Object + + .DESCRIPTION + Improve the speed of looking up a value in an array by creating a hashtable index. + Good for looking up results in very large arrays or CSV files (e.g Import-Csv) + + .NOTES + Version: 1.0 + Author: Miles Gratz + Creation Date: April 10, 2017 + Purpose/Change: Initial script development + + .PARAMETER Array + A mandatory parameter specifying the array used to create a 'FastLookup' (hashtable index) + + .PARAMETER Table + A mandatory parameter specifying the 'FastLookup' created by New-FastLookup + + .PARAMETER Value + A mandatory parameter specifying the search criteria (e.g. "Server458") + + .OUTPUTS + The object(s) in the array that match the search + + .EXAMPLE + + PS> $array = 1..10000000 + PS> $hashtable = New-FastLookup $array + + PS> Measure-Command { + $array | Where-Object { $_ -eq 199999 } + } + + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 9 + Milliseconds : 306 + + PS> Measure-Command { + Get-FastLookup -Value 199999 -Array $array -Table $hashtable + } + + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 0 + Milliseconds : 65 + + [NOTE] Performance test on Windows 10 x64 (i5-6200U, 8GB RAM, SSD) + + #> + param( + [Parameter(Mandatory=$true)] + $Value, + [Parameter(Mandatory=$true)] + [Array]$Array, + [Parameter(Mandatory=$true)] + [Hashtable]$Table + ) + + Try + { + # Lookup Value in hashtable + $Index = $Table[$Value] + + # Find quantity of Index values + $IndexQty = ($Index -split ",").Count + + # Find objects in Array based on Index + # (if multiple, split into array) + If ($IndexQty -eq 1){ $Array[$Index] } + If ($IndexQty -ge 2){ $Array[$Index -split ","] } + + } + Catch + { + $null + } +} \ No newline at end of file diff --git a/New-FastLookup.ps1 b/New-FastLookup.ps1 new file mode 100644 index 0000000..13852ad --- /dev/null +++ b/New-FastLookup.ps1 @@ -0,0 +1,121 @@ +function New-FastLookup { + <# + .SYNOPSIS + Lookup a value in an array faster than Where-Object + + .DESCRIPTION + Improve the speed of looking up a value in an array by creating a hashtable index. + Good for looking up results in very large arrays or CSV files (e.g Import-Csv) + + .NOTES + Version: 1.0 + Author: Miles Gratz + Creation Date: April 10, 2017 + Purpose/Change: Initial script development + + .PARAMETER Array + A mandatory parameter specifying input array used to create 'FastLookup' + + .PARAMETER Header + An optional parameter specifying the header in the input array used to create 'FastLookup' + + .OUTPUTS + A hashtable, listing the values in the array and their corresponding index + + .EXAMPLE + + PS> $array = 1..10000000 + PS> $hashtable = New-FastLookup $array + + PS> Measure-Command { + $array | Where-Object { $_ -eq 199999 } + } + + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 9 + Milliseconds : 306 + + PS> Measure-Command { + Get-FastLookup -Value 199999 -Array $array -Table $hashtable + } + + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 0 + Milliseconds : 65 + + [NOTE] Performance test on Windows 10 x64 (i5-6200U, 8GB RAM, SSD) + + #> + param( + [Parameter(Mandatory=$true)] + [array]$Array, + $Header + ) + + # Identify headers in input array + $Headers = $Array | Get-Member -MemberType 'NoteProperty' | Select-Object -ExpandProperty 'Name' + + # Define empty hashtable and index + $HashTable = @{} + $Index = 0 + + #1: Header specified + #2: Header exists in array + If (($Header -ne $null) -and ($Header -in $Headers)) + { + # Redefine array with only data from specified Header + $Array = $Array.$Header + } + + #1: Header is specified + #2: Header does NOT exist in array + ElseIf (($Header -ne $null) -and ($Header -notin $Headers)) + { + # Exit function with error + Write-Error "Specified header ($Header) does not exist in input array." + Break + } + + #1: Header is NOT specified + #2: Array contains multiple Headers + ElseIf (($Header -eq $null) -and ($Headers.Count -gt 1)) + { + # Exit function with error + Write-Error "Input array requires the -Header parameter (multiple columns detected)." + Break + } + + # Loop through array + foreach ($Item in $Array) + { + # Add index of Item to hashtable + # Name Value + # ---- ----- + # Server1 953 + # Server2 1157 + Try + { + $HashTable.Add($Item,$Index) + } + + # Duplicate key detected, add to existing value + # Name Value + # ---- ----- + # Server1 953 + # Server2 1157,3325 + Catch + { + $HashTable[$Item] = ($HashTable[$Item],$Index -join ",") + } + + # Increment loop + $Index++ + } + + # Output results + $HashTable +} diff --git a/README.md b/README.md index 8ff7f4d..4440c5e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,101 @@ # FastLookup + PowerShell Module designed to optimize the speed of searching an array. + +## How It Works? + +The FastLookup module creates a hashtable, combining the Values in an array with the Index of the item in the array. Since the Name (or "Key") of the hashtable must be unique, each duplicate result is appended in the hashtable: + + PS> $array + + Laptop + Workstation + Laptop + Router + + PS> $hashtable + + Name Value + ---- ----- + Laptop 0,2 + Workstation 1 + Router 3 + +The larger the array, the more significant the performance increase from using FastLookup instead of Where-Object. Here are some speed differences when looking for an object in an array using Where-Object vs FastLookup (hashtable index) method: + + Array with 10000 rows (+6 columns): + + Where-Object: 145ms + FastLookup: 25ms + + Array with 50000 rows (+6 columns): + + Where-Object: 973ms + FastLookup: 53ms + +## Example (1 Column Array) + + PS> $array = 1..1000000 + PS> $hashtable = New-FastLookup $array + + PS> Measure-Command { + $array | Where-Object { $_ -eq 199999 } + } + + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 9 + Milliseconds : 306 + + PS> Measure-Command { + Get-FastLookup -Value 199999 -Array $array -Table $hashtable + } + + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 0 + Milliseconds : 65 + + +## Example (2+ Column Array) + + PS> $array + + DeviceName : Device38 + Type : Security Camera + IP : 10.10.128.1 + VLAN : VLAN85 + Location : Los Angeles + Patch Window : Sunday + + DeviceName : Device68 + Type : Server + IP : 10.10.128.126 + VLAN : VLAN15 + Location : Chicago + Patch Window : Sunday + + PS> $Hashtable = New-FastLookup -Array $Array -Header "Location" + PS> Get-FastLookup -Value "Chicago" -Array $Array -Table $Hashtable + + DeviceName : Device68 + Type : Server + IP : 10.10.128.126 + VLAN : VLAN15 + Location : Chicago + Patch Window : Sunday + +## Install + +Download [FastLookup.zip](https://github.com/milesgratz/FastLookup/releases/download/v1.0/FastLookup.zip) and extract the contents into `C:\Users\[User]\Documents\WindowsPowerShell\modules\Stretch` (you may have to create these directories if they don't exist.) + +## Releases +* [v1.0](https://github.com/milesgratz/FastLookup/releases/download/v1.0/FastLookup.zip) + +## Contributors +* [Miles Gratz](https://github.com/milesgratz) + +## Contact +* Blog: [serveradventures.com](http://www.serveradventures.com) diff --git a/Test-FastLookup.ps1 b/Test-FastLookup.ps1 new file mode 100644 index 0000000..f0350d8 --- /dev/null +++ b/Test-FastLookup.ps1 @@ -0,0 +1,170 @@ +function Test-FastLookup { + <# + .SYNOPSIS + Lookup a value in an array faster than Where-Object + + .DESCRIPTION + Improve the speed of looking up a value in an array by creating a hashtable index. + Good for looking up results in very large arrays or CSV files (e.g Import-Csv) + + .NOTES + Version: 1.0 + Author: Miles Gratz + Creation Date: April 10, 2017 + Purpose/Change: Initial script development + + .OUTPUTS + 1. Creates a sample array + 2. Compares speed of Where-Object vs Get-FastLookup (using Measure-Command) + + .EXAMPLE + + PS > Test-FastLookup + + Creating 'speedtest' array, total quantity: 50000 + Example object: + + DeviceName : Device38 + Type : Security Camera + IP : 10.10.128.1 + VLAN : VLAN85 + Location : Los Angeles + Patch Window : Sunday + + Creating hashtable: + $Hashtable = New-FastLookup -Array $Array -Header "Location" + + -------------------------------------------------------------------------------- + Measuring speed of Where-Object lookup: + + $Array | Where-Object { $_.Location -eq "Chicago" } + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 0 + Milliseconds : 971 + + Number of results: 16468 + First result: + + DeviceName : Device1 + Type : Network Switch + IP : 10.10.128.47 + VLAN : VLAN90 + Location : Chicago + Patch Window : Sunday + + -------------------------------------------------------------------------------- + Measuring speed of Get-FastLookup lookup: + + Get-FastLookup -Value "Chicago" -Array $array -Table $hashtable + Days : 0 + Hours : 0 + Minutes : 0 + Seconds : 0 + Milliseconds : 71 + + Number of results: 16468 + First result: + + DeviceName : Device1 + Type : Network Switch + IP : 10.10.128.47 + VLAN : VLAN90 + Location : Chicago + Patch Window : Sunday + + [NOTE] Performance test on Windows 10 x64 (i5-6200U, 8GB RAM, SSD) + + #> + param( + # Default sample size of 50K + [int]$Quantity = 50000 + ) + + # Define empty array and index + $Array = @() + $Index = 0 + + # Create array based on quantity + Write-Output "Creating 'speedtest' array, total quantity: $Quantity" + while ($Index -lt $Quantity) + { + # Calculate percentage + $PercentageIndex = $Percentage + $Percentage = [math]::Round((($Index / $Quantity)*100)) + + # Announce percentage + If ($Percentage -gt $PercentageIndex) + { + Write-Output "Creating 'speedtest' array ($Percentage%)" + } + + # Example template + $DeviceNumber = 1..100 | Get-Random + $Type = Get-Random 'Laptop','Workstation','Network Switch','Wireless Access Point','Security Camera','Server' + $IP = "10.10.128." + (Get-Random (1..254)) + $VLAN = "VLAN" + (Get-Random (1..100)) + $Location = Get-Random 'Chicago','New York','Los Angeles' + $PatchWindow = Get-Random 'Sunday','Saturday' + + # Create custom object to add to array + $Object = New-Object PSCustomObject + $Object | Add-Member -MemberType NoteProperty -name "DeviceName" -Value "Device$DeviceNumber" + $Object | Add-Member -MemberType NoteProperty -name "Type" -Value "$Type" + $Object | Add-Member -MemberType NoteProperty -name "IP" -Value "$IP" + $Object | Add-Member -MemberType NoteProperty -name "VLAN" -Value "$VLAN" + $Object | Add-Member -MemberType NoteProperty -name "Location" -Value "$Location" + $Object | Add-Member -MemberType NoteProperty -name "Patch Window" -Value "$PatchWindow" + + # Add object to array + $Array += $Object + + # Increment loop + $Index++ + } + + # Example object + Write-Output "" + Write-Output "Example object:" + Write-Output $Array[0] + + # Creating New-FastLookup hashtable + Write-Output "" + Write-Output "Creating hashtable: " + Write-Output "" + Write-Output ('$Hashtable = New-FastLookup -Array $Array -Header "Location"') + $Hashtable = New-FastLookup -Array $Array -Header "Location" + + # Measuring speed of Where-Object lookup + Write-Output "" + Write-Output ('-'*80) + Write-Output 'Measuring speed of Where-Object lookup:' + Write-Output "" + Write-Output ('$Array | Where-Object { $_.Location -eq "Chicago" }') + $MeasureWhereObject = Measure-Command { + $WhereObjectResults = $Array | Where-Object { $_.Location -eq "Chicago" } + } | Select Days,Hours,Minutes,Seconds,Milliseconds + Write-Output $MeasureWhereObject + Write-Output "" + Write-Output "Number of results: $($WhereObjectResults.Count)" + Write-Output "First result:" + Write-Output "" + Write-Output $WhereObjectResults[0] + + # Measuring speed of Get-FastLookup + Write-Output ('-'*80) + Write-Output 'Measuring speed of Get-FastLookup lookup:' + Write-Output "" + Write-Output ('Get-FastLookup -Value "Chicago" -Array $array -Table $hashtable') + $MeasureFastLookup = Measure-Command { + $FastLookupResults = Get-FastLookup -Value "Chicago" -Array $Array -Table $Hashtable + } | Select Days,Hours,Minutes,Seconds,Milliseconds + Write-Output "" + Write-Output $MeasureFastLookup + Write-Output "" + Write-Output "Number of results: $($FastLookupResults.Count)" + Write-Output "First result:" + Write-Output "" + Write-Output $FastLookupResults[0] +}