diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..050960e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +ptfrFiles/test.results/* diff --git a/README.md b/README.md index f162303..bcfe449 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,76 @@ -# ptrFiles -Protect Transfer Reconcile Files +# ptrFiles - Protect, Transfer, Reconcile Files + +## Summary + +ptrFiles is for Protecting, Transfering and Reconciling Files on remote computer +where the computers are isolated or on different networks. + +The process uses a Windows PowerShell script and both the source and target computers +that execute the code are required to be installed with Windows PowerShell. + +The folder contents at source are archived and encrypted into a single file. You +transfer the file to your target, where the content are unpacked using the decryption +key. After archive contents are restored you can execute the reconcile function +to veriy that the contents are transferred, unaltered. + +If you have access to both source and target folders, then you should consider +using tools such as: + +* Microsoft ROBOCOPY +* rsync + +Alternatively, you can use backup and restore utilities on the folder, and rely that +the contents are restored correctly. If you want this to be secure, ensure +the backup is encrypted. + +**Note**: If you require reconciliation (comparison) of files between the source +and target, then you may be required to use additional software. An example is +JAM Software FileList. + +**Note**: Disk size utilities are not suitable for transferring/copying content + +## Background + +The script was born out a necessity to transfer a large volume of photographs +from one server to another, where shared network drives was not a feasible +solution. + +## Usage + +Packages source folder contents into a 7ZIP file, adding a reconciliation +file to the 7ZIP file and then encrypting the contents. Send + +* this script +* the 7ZIP package file +* plus optional SecretFilename ( if using RecipientKeyName ) to the target or recipient. + +The source folder is not altered and only read rights are required. A log +file is written at exceution to record activity. + +The SecretFileName can be sent via email, while the 7ZIP can go different routes +due to possible size such as: + +* Cloud storage provider +* HTTPS web file upload +* SFTP transfer +* USB stick + +At the target, unpack the contents to a folder and reconcile the results. You +will need write access on the target storage. A log file is written at exceution +to record activity. + +Your bulk file transfer is encrypted in transit. Note that if you use the +SecretKey method the ecnrypted contents will only be as secure as the strength +of your secret. + +You can use storage providers such as Dropbox, AWS S3, Google Drive, OneDrive or BackBlaze +and your documents have additonal protection. + +A log file is produced on execution. Repeated executions on the same day +will add text content to the same log file. The default log name takes the form: +"ptr_files_yyyy-MM-dd.log" + +You will need to have installed the 7Zip4Powershell PowerShell cmdlet +before using the pack or unpack actions. You can install the cmdlet +by executing +.\ptrFiles.ps1 -Action install -Path ".\" diff --git a/ptfrFiles/ptrFiles.ps1 b/ptfrFiles/ptrFiles.ps1 index 58cf745..17d6d21 100644 --- a/ptfrFiles/ptrFiles.ps1 +++ b/ptfrFiles/ptrFiles.ps1 @@ -46,11 +46,12 @@ .Parameter Action Action to perform, which can be: - - Install - - Pack - - Unpack - - Reconcile - - ReconcileFile + - Install : Install 7Zip4PowerShell + - Pack : Archive the contents of a folder(s) + - Unpack : Unpack the archive, but no reconfile is performed + - Reconcile : Reconcile the contents in the restored folder + - ReconcileFile : Generate reconfile file. The pack process does this. + - ArchiveInformation : Fetch archive information .Parameter Path The path to the files and folders to pack or the path to the unpack location. @@ -59,7 +60,13 @@ When using the trailing * for names, the filtering is only applied to immediate folder names under the parent folder. The filter does not cascade to lower folders. - + + The Path can also be a file containing a list of paths, one per line. To use a + list file, prefix the Path value with a "@" and name the file. Do not use a folder + for @ defined path. + + A file (@ prefix) containing a list of paths cannot contain generic path names, that + is paths with trailing wildcard of "*" .Parameter RecipientKeyName The recipient of the package which is used to find the appropriate @@ -80,6 +87,8 @@ uses a symmetric cryptographic key exchange which is less secure then the RecipientKeyName approach. + Note: Currently the script doe snot user Secure Strings + .Parameter ArchiveFileName The location and name of the 7ZIP file. If not supplied a default 7ZIP file name will be generated in the current directory. @@ -337,31 +346,33 @@ Param( Write-Host "Using @ file '$($folderName.Substring(1))'" Get-Content -Path $($folderName.Substring(1)) | ForEach-Object { - If (!(Test-Path -Path $_ )) { - Write-Log "Folder/file '$($_)' does not exist" - Write-Host "Folder/file '$($_)' does not exist" -ForegroundColor Red - } - else { - Get-ChildItem $_ -Filter $fileFilter -Recurse | Where-Object {!$_.PSIsContainer} | ForEach-Object { + if ($_ -ne "") { + If (!(Test-Path -Path $_ )) { + Write-Log "Folder/file '$($_)' does not exist" + Write-Host "Folder/file '$($_)' does not exist" -ForegroundColor Red + } + else { + Get-ChildItem $_ -Filter $fileFilter -Recurse | Where-Object {!$_.PSIsContainer} | ForEach-Object { - $totalFilecount = $totalFileCount + 1 - $totalFileSize = $totalFileSize + $_.Length - - if (($totalFilecount % $messageFrequency) -eq 0) { - Write-Log "Read $totalFileCount files and size $(Get-ConvenientFileSize -Size $totalFileSize ). Currently at folder '$($_.Directory)' " - Write-Host "Read $totalFileCount files and size $(Get-ConvenientFileSize -Size $totalFileSize ). Currently at folder '$($_.Directory)' " + $totalFilecount = $totalFileCount + 1 + $totalFileSize = $totalFileSize + $_.Length + + if (($totalFilecount % $messageFrequency) -eq 0) { + Write-Log "Read $totalFileCount files and size $(Get-ConvenientFileSize -Size $totalFileSize ). Currently at folder '$($_.Directory)' " + Write-Host "Read $totalFileCount files and size $(Get-ConvenientFileSize -Size $totalFileSize ). Currently at folder '$($_.Directory)' " + } + + if ($ExcludeHash) { + $sourceHash = "" + } else { + $sourceHash = (Get-FileHash -Path $_.FullName).Hash + } + $record = '"'+$_.FullName.Replace($rootFolderName, "")+'","'+$_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")+'"' + $record = $record + ',"'+$_.CreationTime.ToString("yyyy-MM-ddTHH:mm:ss")+'","'+$_.LastAccessTime.ToString("yyyy-MM-ddTHH:mm:ss")+'"' + $record = $record + ','+$_.Length+',"'+$sourceHash+'","'+ $_.Directory + '","' + $_.Name + '","' + $_.Attributes+'","'+$_.Extension+'"' + Add-Content -Path $reconcileFile -Value $record + } - - if ($ExcludeHash) { - $sourceHash = "" - } else { - $sourceHash = (Get-FileHash -Path $_.FullName).Hash - } - $record = '"'+$_.FullName.Replace($rootFolderName, "")+'","'+$_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")+'"' - $record = $record + ',"'+$_.CreationTime.ToString("yyyy-MM-ddTHH:mm:ss")+'","'+$_.LastAccessTime.ToString("yyyy-MM-ddTHH:mm:ss")+'"' - $record = $record + ','+$_.Length+',"'+$sourceHash+'","'+ $_.Directory + '","' + $_.Name + '","' + $_.Attributes+'","'+$_.Extension+'"' - Add-Content -Path $reconcileFile -Value $record - } } } @@ -458,7 +469,6 @@ Param( } Write-Log "Saving folders/files to archive file '$compressFile'" - Write-Log "Source folder is '$transferFolder'" Write-Host "Saving folders/files to archive file '$compressFile'" if ($reconcileFile -eq "") @@ -473,6 +483,7 @@ Param( if ($transferFolder.EndsWith("*")) { + Write-Log "Archive primary folder is '$transferFolder'" $firstCompress = $true Get-ChildItem $transferFolder| ForEach-Object { @@ -501,28 +512,30 @@ Param( Write-Host "Using @ file '$($transferFolder.Substring(1))'" Get-Content -Path $($transferFolder.Substring(1)) | ForEach-Object { - If (!(Test-Path -Path $_ )) { - Write-Log "Folder/file '$($_)' does not exist" - Write-Host "Folder/file '$($_)' does not exist" -ForegroundColor Red - } - else { - Write-Log "Archive folder '$($_)'" - Write-Host "Archivefolder '$($_)'" - if (Test-Files -FolderName $_ -FileFilter $fileFilter) { - try { - if ($firstCompress) { - Compress-7Zip -Path $_ -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Filter $fileFilter - } else { - Compress-7Zip -Path $_ -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Filter $fileFilter -Append + if ($_ -ne "") { + If (!(Test-Path -Path $_ )) { + Write-Log "Folder/file '$($_)' does not exist" + Write-Host "Folder/file '$($_)' does not exist" -ForegroundColor Red + } + else { + Write-Log "Archive folder '$($_)'" + Write-Host "Archivefolder '$($_)'" + if (Test-Files -FolderName $_ -FileFilter $fileFilter) { + try { + if ($firstCompress) { + Compress-7Zip -Path $_ -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Filter $fileFilter + } else { + Compress-7Zip -Path $_ -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Filter $fileFilter -Append + } + $firstCompress = $false + } catch { + Write-Log "Compress error with file '$($_)'. See any previous errors. $Error" + Write-Host "Compress error with file '$($_)'. See any previous errors. $Error" -ForegroundColor Red } - $firstCompress = $false - } catch { - Write-Log "Compress error with file '$($_)'. See any previous errors. $Error" - Write-Host "Compress error with file '$($_)'. See any previous errors. $Error" -ForegroundColor Red + } else { + Write-Log "Empty folder '$($_.FullName)'" + Write-Host "Empty folder '$($_.FullName)'" } - } else { - Write-Log "Empty folder '$($_.FullName)'" - Write-Host "Empty folder '$($_.FullName)'" } } } @@ -590,7 +603,8 @@ function Invoke-Reconcile Param( [Parameter(Mandatory)][String] $ReconcileFile, [Parameter(Mandatory)][String] $Folder, - [String] $TargetReconcileFile + [String] $TargetReconcileFile, + [Switch] $ExtendedCheck ) if ($reconcileFile -eq "") @@ -617,6 +631,7 @@ Param( $totalFileCount = 0 $totalFileSize = 0 $errorCount = 0 + $missingFileCount = 0 $missingHash = $false # For each entry in the reconcile file @@ -629,17 +644,35 @@ Param( $targetHash= (Get-FileHash -Path $restoreFileName).Hash if ($_.Hash -ne $targetHash) { $errorCount = $errorCount + 1 - Write-Log "Hash mismatch for file '$restoreFileName'" + Write-Log "Hash mismatch for file '$restoreFileName' with target value $targetHash" } } else { $missingHash = $true } - if ((Get-Item -Path $restoreFileName).LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.LastWriteTime) { + if ((Get-Item -Path $restoreFileName).CreationTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.CreationTime) { $errorCount = $errorCount + 1 - Write-Log "Last write mismatch for file '$restoreFileName'" + Write-Log "Creation mismatch for file '$restoreFileName' with target value $((Get-Item -Path $restoreFileName).CreationTime.ToString("yyyy-MM-ddTHH:mm:ss"))" } + if ((Get-Item -Path $restoreFileName).Length -ne $_.Length) { + $errorCount = $errorCount + 1 + Write-Log "Length mismatch for file '$restoreFileName' with target value $(Get-Item -Path $restoreFileName).Length)" + } + + # Note that last / write access time is not checked by default as it will comonly be changed after restore + if ($extendedCheck) { + if ((Get-Item -Path $restoreFileName).LastAccessTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.LastAccessTime) { + $errorCount = $errorCount + 1 + Write-Log "Last access mismatch for file '$restoreFileName' with target value $((Get-Item -Path $restoreFileName).LastAccessTime.ToString("yyyy-MM-ddTHH:mm:ss"))" + } + if ((Get-Item -Path $restoreFileName).LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.LastWriteTime) { + $errorCount = $errorCount + 1 + Write-Log "Last write mismatch for file '$restoreFileName' with target value $((Get-Item -Path $restoreFileName).LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss"))" + } + } + $totalFileSize = $totalFileSize + (Get-Item -Path $restoreFileName).Length } else { + $missingFileCount = $missingFileCount + 1 $errorCount = $errorCount + 1 Write-Log "Non existant target file '$restoreFileName'" } @@ -648,17 +681,23 @@ Param( Write-Log "Total file storage size is $(Get-ConvenientFileSize -Size $totalFileSize ) ($totalFileSize)" Write-Host "Total file storage size is $(Get-ConvenientFileSize -Size $totalFileSize )" - Write-Log "Total file count is $totalFileCount with $errorCount errors" if ($missingHash) { Write-Log "Reconcile file had one or many or all blank hash entries" Write-Host "Reconcile file had one or many or all blank hash entries" -ForegroundColor Yellow } + + Write-Log "Total file count is $totalFileCount with $errorCount errors" + Write-Log "There are $missingFileCount missing files" + if ($errorCount -gt 0) { Write-Host "Total file count is $totalFileCount with $errorCount errors" -ForegroundColor Red } else { Write-Host "Total file count is $totalFileCount with $errorCount errors" -ForegroundColor Green } + if ($missingFileCount -gt 0) { + Write-Host "There are $missingFileCount missing files" -ForegroundColor Red + } } $dateTimeStart = Get-Date -f "yyyy-MM-dd HH:mm:ss" @@ -770,17 +809,43 @@ if ($action -eq "Reconcile") { Invoke-Reconcile -ReconcileFile $localReconcileFile -Folder $path } +if ($action -eq "ArchiveInformation") { + $actioned = $true + if (($RecipientKeyName -eq "") -and ($SecretKey -eq "")) { + Write-Log "Recipient Key Name or Secret Key required for 7Zip information" + Write-Host "Recipient Key Name or Secret Key required for 7Zip information" -ForegroundColor Red + Close-Log + return + } + + if ($SecretKey -eq "") { + if ($secretFileName -eq "") + { + $secretFileName = $default_secretEncrypted + } + $secret = Unprotect-CmsMessage -To $recipientKeyName -Path $secretFileName + } else { + $secret = $SecretKey + } + Write-Log "Retrieving archive information" + Write-Host "Retrieving archive information" + + Get-7ZipInformation -ArchiveFileName $ArchiveFileName -Password $secret +} + if (!($actioned)) { Write-Log "Unknown action '$action'. No processing performed" Write-Host "Unknown action '$action'. No processing performed" -ForegroundColor Red Write-Host "Recognised actions: " - Write-Host " Pack : Pack folder contents into secure 7Zip file" - Write-Host " Unpack : Unpack folder contents from secure 7Zip file" - Write-Host " Reconcile : Reconcile files in unpack folder with list of packed files" - Write-Host " ReconcileFile : Generate a reconcile file without packing" - Write-Host " Install : Install required packages" + Write-Host " Pack : Pack folder contents into secure 7Zip file" + Write-Host " Unpack : Unpack folder contents from secure 7Zip file" + Write-Host " Reconcile : Reconcile files in unpack folder with list of packed files" + Write-Host " ReconcileFile : Generate a reconcile file without packing" + Write-Host " Install : Install required packages" + Write-Host " ArchiveInformation : Fetch archive information from archive file" + Write-Host "" Write-Host "For help use command " Write-Host " Get-Help .\ptrFiles.ps1"