From 50e7bc2250fb638bbee1b22dde8c431f98c30d31 Mon Sep 17 00:00:00 2001 From: Tom Peltonen Date: Sat, 10 Jul 2021 14:27:55 +1000 Subject: [PATCH] Code fixes --- ptfrFiles/ptrFiles.ps1 | 354 +++++++++++++++++++++++++++++++++++------ 1 file changed, 301 insertions(+), 53 deletions(-) diff --git a/ptfrFiles/ptrFiles.ps1 b/ptfrFiles/ptrFiles.ps1 index 0f45f4a..1dc6032 100644 --- a/ptfrFiles/ptrFiles.ps1 +++ b/ptfrFiles/ptrFiles.ps1 @@ -1,4 +1,159 @@ +<# + .Synopsis + Allows the secure transfer and reconciliation of a large number of files + .Description + Packages source folder contents into a 7ZIP file, adding a reconciliation + file to the 7ZIP file and then encrypting the contents. Send + * this script + * the 7ZIP package file + * plus optional SecretFilename ( if using RecipientKeyName ) + to the target or recipient. + + The source folder is not altered and only read rights are required. A log + file is written at exceution to record activity. + + The SecretFileName can be sent via email, while the 7ZIP can go different routes + such as: + * HTTPS web file upload + * SFTP transfer + * USB stick + + At the target, unpack the contents to a folder and reconcile the results. You + will need write access on the target storage. A log file is written at exceution + to record activity. + + Your bulk file transfer is encrypted in transit. Note that if you use the + SecretKey method the ecnrypted contents will only be as secure as the strength + of your secret. + + A log file is produced on execution. Repeated executions on the same day + will add text content to the same log file. The default log name takes the form: + "protect_transfer_reconcile_yyyy-MM-dd.log" + + You will need to have installed the 7Zip4Powershell PowerShell cmdlet + before using the pack or unpack actions. You can install the cmdlet + by executing + .\protect_transfer_reconcile.ps1 -Action install -Path ".\" + + Author: Tom Peltonen + + .Parameter Action + Action to perform, which can be: + - Install + - Pack + - Unpack + - Reconcile + - ReconcileFile + + .Parameter Path + The path to the files and folders to pack or the path to the unpack location. + The path can include a trailing * as a wildcard to only include a subset of + directories. + + When using the trailing * for names, the filtering is only applied to immediate + folder names under the parent folder. The filter does not cascade to lower folders. + + + .Parameter RecipientKeyName + The recipient of the package which is used to find the appropriate + certificate for encrypting with the public key. Either the RecipientKeyName + or the SecretKey is required for packing or unpacking the 7ZIP file. + Using the RecipientKeyName is the most secure transfer option as a + asymmetric cryptographic key is used that can only be decrypted by the + holder of the private key. + + If you are using the RecipientKeyName, then the 7ZIP file contents can only + be unzipped by the holder of the private key and the SecretFileName file. + If you don't have the private, which you should not unless you are sending + to yourself, then you cannot unpack the 7ZIP file. + + .Parameter SecretKey + A tradiitional secret to encrypt or decrypt the 7ZIP package. Either the RecipientKeyName + or the SecretKey is required for packing or unpacking the 7ZIP file. This method + uses a symmetric cryptographic key exchange which is less secure then the + RecipientKeyName approach. + + .Parameter TransferFileName + The location and name of the 7ZIP file. If not supplied a default 7ZIP file name + will be generated in the current directory. + + The default name will take the form ".\transfer_protect_yyyyMMdd_hhmm.7z" + + .Parameter RootFolderName + The root folder, which should be used if using wildcard (*) for the + path. A guess will be made as to value if not supplied, which will + work in many circumstances. + + .Parameter ReconcileFileName + The name of the reconfile file name to generate during pack or use + during unpack. This is a file name without path. If no value is + supplied, then a default name is generated. + The reconcile file is included into the root of the 7ZIP file. + Once a reconcile is executed, you can delete this file from the + restored location. + + The default name is "##protect_transfer_reconcile##.csv" + + .Parameter SecretFileName + The secret file name is used with RecipientKeyName to secure the + internally generated password for the 7ZIP file. When unpacking the + 7ZIP file you will need access to this file if RecipientKeyName + was used. If not supplied a default name is used. This file is + encrypted with RecipientKeyName. + + The default name is ".\transfer.key" + + .Parameter ExcludeHash + Exclude the file hash from the reconcile file. As producing a file + hash takes compute cycles during pack, you can select to bypass this + generation to speed up the packaging. Excluding the hash does reduce + the functionality of the reconciliation at unpack. + + .Parameter LogPath + The log folder where log files are written. If the folder does not + exist then it is created. You need write access rights to this location. + + .Notes + This script has been written to use the 7ZIP function as it is open source + and provides a secure encryption mechanism, plus portability on Windows, + Linux and MacOS. + + It is also beneficial that 7ZIP has efficient compression algorithms. + + Compressing and packing a large data set can take significant time and also + require storage space. The script does not check if you have sufficient + free storage to package the source contents into a single 7ZIP file. It is your + responsibility to ensure sufficient storage space exists. + + + .Example + # Pack and encrypt all files in folder ".\transferpack\" using a private-public key + # A file named ".\transfer.key" is also generated alongside the 7ZIP file + protect_transfer_reconcile.ps1 -Action pack -Path ".\transferpack\" -RecipientKeyName data@mycompany + + .Example + # Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a private-public key + # You will need the file named ".\transfer.key" to unpack the encrypted 7ZIP file + .\protect_transfer_reconcile.ps1 -Action unpack -TransferFileName "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKeyName data@mycompany + + .Example + # Reconcile files in folder ".\targetdir" + .\protect_transfer_reconcile.ps1 -Action reconcile -Path ".\targetdir" + + .Example + # Pack and encrypt all files in folder ".\transferpack\" using a password + .\protect_transfer_reconcile.ps1 -Action pack -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds" + + .Example + # Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a password + .\protect_transfer_reconcile.ps1 -Action unpack -TransferFileName "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds" + + .Example + # Pack and encrypt all files in folder ".\transferpack\02*" where the folder name starts with "02" using a password + .\protect_transfer_reconcile.ps1 -Action pack -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds" + +#> param ( [Parameter(Mandatory)][String] $Action, @@ -6,9 +161,11 @@ param ( [String] $RecipientKeyName, [String] $SecretKey, [String] $TransferFileName, + [String] $RootFolderName, [String] $ReconcileFileName, [String] $SecretFileName, - [switch] $ExcludeHash + [switch] $ExcludeHash, + [String] $LogPath ) @@ -24,7 +181,10 @@ function Write-Log { $date = Get-Date -f "yyyy-MM-dd" - $logPath = Join-Path -Path ".\" -ChildPath "Logs" + if ($LogPath -eq "") + { + $logPath = Join-Path -Path ".\" -ChildPath "Logs" + } $logName = "protect_transfer_reconcile_$date.log" $sFullPath = Join-Path -Path $logPath -ChildPath $logName @@ -64,6 +224,19 @@ param( } } +function Test-Files +{ + Param( + [Parameter(Mandatory)][String] $FolderName, + [String] $Filter + ) + + Get-ChildItem $folderName -Recurse | Where-Object {!$_.PSIsContainer} | ForEach-Object { + return $true + } + + return $false +} # Reconcile function Set-Reconcile @@ -71,6 +244,7 @@ function Set-Reconcile Param( [Parameter(Mandatory)][String] $ReconcileFile, [Parameter(Mandatory)][String] $FolderName, + [String] $RootFolderName, [String] $Filter, [switch] $Feedback = $false ) @@ -84,7 +258,7 @@ Param( Write-Log "Folder '$folderName' does not exist" Write-Host "Folder '$folderName' does not exist" -ForegroundColor Red Close-Log - return + Exit } Write-Log "Generating reconciliation file '$reconcileFile'" @@ -93,8 +267,12 @@ Param( $totalFileCount = 0 $totalFileSize = 0 + if ($rootFolderName -eq "") { + $rootFolderName = $folderName + } + Set-Content -Path $reconcileFile -Value '"FullName","LastWriteTime","Length","Hash","ParentFolder","Object"' - Get-Childitem $folderName -Recurse | Where-Object {!$_.PSIsContainer} | ForEach-Object { + Get-ChildItem $folderName -Recurse | Where-Object {!$_.PSIsContainer} | ForEach-Object { $totalFilecount = $totalFileCount + 1 $totalFileSize = $totalFileSize + $_.Length if ($ExcludeHash) { @@ -102,7 +280,7 @@ Param( } else { $sourceHash = (Get-FileHash -Path $_.FullName).Hash } - $record = '"'+$_.FullName.Replace($folderName, "")+'","'+$_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")+'",'+$_.Length+',"'+$sourceHash+'","'+ $_.Directory + '","' + $_.Name + '"' + $record = '"'+$_.FullName.Replace($rootFolderName, "")+'","'+$_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")+'",'+$_.Length+',"'+$sourceHash+'","'+ $_.Directory + '","' + $_.Name + '"' Add-Content -Path $reconcileFile -Value $record } @@ -124,31 +302,31 @@ Param( } } - Write-Log "Total reconcile file count is $totalFileCount and size $totalFileXbytes $totalRightLabel" + Write-Log "Total reconcile file count is $totalFileCount and size $totalFileXbytes $totalRightLabel ($totalFileSize)" if ($feedback) { Write-Host "Total reconcile file count is $totalFileCount and size $totalFileXbytes $totalRightLabel" -ForegroundColor Green } } - +# Compress / Package function Invoke-Pack { Param( [String] $TransferFolder, + [String] $RootFolder, [String] $Filter, [String] $Secret, [String] $CompressFile, [String] $ReconcileFile ) - # Send + If (!(Test-Path -Path $transferFolder )) { Write-Log "Folder '$transferFolder' does not exist" Write-Host "Folder '$transferFolder' does not exist" -ForegroundColor Red Close-Log - return + Exit } - Write-Log "Packing files to compress file '$compressFile'" Write-Log "Source folder is '$transferFolder'" Write-Host "Packing files to compress file '$compressFile'" @@ -158,29 +336,56 @@ Param( $reconcileFile = $default_reconcileFile } + if ($transferFolder.EndsWith("*")) + { + $firstCompress = $true -# $Files = Get-ChildItem -Path “D:\Logs_folder” -Filter “*.txt” -Recurse -File | Where-Object {$_.LastWriteTime -le $LastWrite} - - $firstPack = $true - Write-Host "Folder $transferFolder" - Get-ChildItem -Path $transferFolder -Filter "*20*" -Recurse | Where-Object {$_.PSIsContainer} | ForEach-Object { - Write-Log "File found $($_.FullName)" - if ($firstPack) { - $firstPack = $false - Compress-7Zip -Path $_.FullName -ArchiveFileName $compressFile -Format SevenZip - } else { - Compress-7Zip -Path $_.FullName -ArchiveFileName $compressFile -Format SevenZip -Append + Get-ChildItem $transferFolder| ForEach-Object { + Write-Log "Transfer folder '$($_.FullName)'" + Write-Host "Transfer folder '$($_.FullName)'" + if (Test-Files -FolderName $_.FullName -Filter $filter) { + try { + if ($firstCompress) { + Compress-7Zip -Path $_.FullName -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot + } else { + Compress-7Zip -Path $_.FullName -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Append + } + $firstCompress = $false + } catch { + Write-Log "Compress error with file '$($_.FullName)'. See any previous errors. $Error" + Write-Host "Compress error with file '$($_.FullName)'. See any previous errors. $Error" -ForegroundColor Red + } + } else { + Write-Log "Empty folder '$($_.FullName)'" + Write-Host "Empty folder '$($_.FullName)'" + } } + } else { + Write-Log "Transfer folder '$transferFolder'" + Write-Host "Transfer folder '$transferFolder'" + Compress-7Zip -Path $transferFolder -ArchiveFileName $compressFile -Format SevenZip } -# Compress-7Zip -Path $transferFolder -ArchiveFileName $compressFile -Format SevenZip -Append - #Set-Reconcile -ReconcileFile $reconcileFile -FolderName $transferFolder -Filter $filter + If (!(Test-Path -Path $compressFile )) { + Write-Log "Compress file '$compressFile' was not created. See any previous errors" + Write-Host "Compress file '$compressFile' was not created. See any previous errors" -ForegroundColor Red + Close-Log + Exit + } + + Set-Reconcile -ReconcileFile $reconcileFile -FolderName $transferFolder -Filter $filter -RootFolderName $rootFolder + If (!(Test-Path -Path $compressFile )) { + Write-Log "Reconcile file '$reconcileFile' was not created. See any previous errors" + Write-Host "Reconcile file '$reconcileFile' was not created. See any previous errors" -ForegroundColor Red + Close-Log + return + } Write-Log "Add reconcile file '$reconcileFile' to file '$compressFile'" -# $fullReconcileName = (Get-Item $reconcileFile).FullName -# $fullZipName = (Get-Item $compressFile).FullName -# Compress-7Zip -Path $fullReconcileName -ArchiveFileName $fullZipName -Format SevenZip -Append -Password $secret -EncryptFilenames -# Remove-Item $fullReconcileName + $fullReconcileName = (Get-Item $reconcileFile).FullName + $fullZipName = (Get-Item $compressFile).FullName + Compress-7Zip -Path $fullReconcileName -ArchiveFileName $fullZipName -Format SevenZip -Append -Password $secret -EncryptFilenames + Remove-Item $fullReconcileName Write-Log "Package ready in file '$compressFile' from folder '$transferFolder'" Write-Host "Package ready in file '$compressFile' from folder '$transferFolder'" -ForegroundColor Green @@ -199,7 +404,7 @@ Param( Write-Log "Transfer/compress file '$CompressFile' does not exist" Write-Host "Transfer/compress file '$CompressFile' does not exist" -ForegroundColor Red Close-Log - return + Exit } Write-Log "Restoring files transferred to '$restoreFolder'" @@ -216,58 +421,90 @@ Param( function Invoke-Reconcile { Param( - [Parameter(Mandatory)][String] $reconcileFileName, - [Parameter(Mandatory)][String] $folderName, + [Parameter(Mandatory)][String] $ReconcileFile, + [Parameter(Mandatory)][String] $Folder, + [String] $TargetReconcileFile, [String] $Filter ) - if ($reconcileFileName -eq "") + if ($reconcileFile -eq "") { - $reconcileFileName = $default_reconcileFile + $reconcileFile = $default_reconcileFile } Write-Log "Reconciling documents transferred" Write-Host "Reconciling documents transferred" - If (!(Test-Path -Path $reconcileFileName )) { - Write-Log "Reconciliation file '$reconcileFileName' does not exist" - Write-Host "Reconciliation file '$reconcileFileName' does not exist" -ForegroundColor Red + If (!(Test-Path -Path $reconcileFile )) { + Write-Log "Reconciliation file '$reconcileFile' does not exist" + Write-Host "Reconciliation file '$reconcileFile' does not exist" -ForegroundColor Red Close-Log - return + Exit } - If (!(Test-Path -Path $folderName )) { - Write-Log "Folder '$folderName' does not exist" - Write-Host "Folder '$folderName' does not exist" -ForegroundColor Red + If (!(Test-Path -Path $folder )) { + Write-Log "Folder '$folder' does not exist" + Write-Host "Folder '$folder' does not exist" -ForegroundColor Red Close-Log - return + Exit } - Write-Log "Using reconciliation file '$reconcileFileName'" + Write-Log "Using reconciliation file '$reconcileFile'" - $totalFilecount = 0 + $totalFileCount = 0 + $totalFileSize = 0 $errorCount = 0 + $missingHash = $false # For each entry in the reconcile file # find the file and compare hash - Import-Csv $reconcileFileName | ForEach-Object { + Import-Csv $reconcileFile | ForEach-Object { $totalFileCount = $totalFileCount +1 - $restoreFileName = $(Join-Path -Path $folderName -ChildPath $_.FullName) + $restoreFileName = $(Join-Path -Path $folder -ChildPath $_.FullName) If (Test-Path -Path $restoreFileName ) { - $targetHash= (Get-FileHash -Path $restoreFileName).Hash - if ($_.Hash -ne $targetHash) { - $errorCount = $errorCount + 1 - Write-Log "Hash mismatch for file '$restoreFileName'" + if ($_.Hash -ne "") { + $targetHash= (Get-FileHash -Path $restoreFileName).Hash + if ($_.Hash -ne $targetHash) { + $errorCount = $errorCount + 1 + Write-Log "Hash mismatch for file '$restoreFileName'" + } + } else { + $missingHash = $true } if ((Get-Item -Path $restoreFileName).LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.LastWriteTime) { $errorCount = $errorCount + 1 Write-Log "Last write mismatch for file '$restoreFileName'" } - + $totalFileSize = $totalFileSize + (Get-Item -Path $restoreFileName).Length } else { $errorCount = $errorCount + 1 Write-Log "Non existant target file '$restoreFileName'" } } + if ($totalFileSize -ge 1000000000000) { + $totalRightLabel = "TB" + $totalFileXbytes = [math]::Round(($totalFileSize / 1000000000000), 2) + } else { + if ($totalFileSize -ge 1000000000) { + $totalRightLabel = "GB" + $totalFileXbytes = [math]::Round(($totalFileSize / 1000000000), 2) + } else { + if ($totalFileSize -ge 1000000) { + $totalRightLabel = "MB" + $totalFileXbytes = [math]::Round(($totalFileSize / 1000000), 2) + } else { + $totalRightLabel = "KB" + $totalFileXbytes = [math]::Round(($totalFileSize / 1000), 2) + } + } + } + Write-Log "Total file storage size is $totalFileXbytes $totalRightLabel ($totalFileSize)" + Write-Host "Total file storage size is $totalFileXbytes $totalRightLabel" + Write-Log "Total file count is $totalFileCount with $errorCount errors" + if ($missingHash) + { + Write-Log "Reconcile file had one or many or all blank hash entries" + Write-Host "Reconcile file had one or many or all blank hash entries" -ForegroundColor Yellow + } if ($errorCount -gt 0) { Write-Host "Total file count is $totalFileCount with $errorCount errors" -ForegroundColor Red } else { @@ -296,7 +533,18 @@ if ($action -eq "Pack") { Close-Log return } - + + if ($rootFolderName -eq "") { + if ($path.EndsWith("*")) { + Write-Log "Root folder required for packing when using wild card for Path" + Write-Host "Root folder required for packing when using wild card for Path" -ForegroundColor Red + Close-Log + return + } else { + $rootFolderName = $path + } + } + if ($TransferFileName -eq "") { $TransferFileName = ".\transfer_protect_$default_dateLocal.7z" } @@ -312,7 +560,7 @@ if ($action -eq "Pack") { $secret = $SecretKey } - Invoke-Pack -TransferFolder $path -Secret $secret -CompressFile $transferFileName -ReconcileFile $reconcileFileName + Invoke-Pack -TransferFolder $path -Secret $secret -CompressFile $transferFileName -ReconcileFile $reconcileFileName -RootFolder $rootFolderName } @@ -351,7 +599,7 @@ if ($action -eq "ReconcileFile") { { $reconcileFileName = $default_reconcileFile } - Set-Reconcile -ReconcileFile $reconcileFileName -FolderName $path -Feedback + Set-Reconcile -ReconcileFile $reconcileFileName -FolderName $path -Feedback -RootFolderName $rootFolderName } @@ -362,7 +610,7 @@ if ($action -eq "Reconcile") { $reconcileFileName = $default_reconcileFile } $localReconcileFile = Join-Path -Path $path -ChildPath $reconcileFileName - Invoke-Reconcile -ReconcileFile $localReconcileFile -FolderName $path + Invoke-Reconcile -ReconcileFile $localReconcileFile -Folder $path }