Cleaned code and added ZIP info

README text addded

.gitignore

@@ -0,0 +1 @@
+ptfrFiles/test.results/*

README.md

@@ -1,2 +1,76 @@
-# ptrFiles
+# ptrFiles - Protect, Transfer, Reconcile Files
-Protect Transfer Reconcile Files
+
+## Summary
+
+ptrFiles is for Protecting, Transfering and Reconciling Files on remote computer
+where the computers are isolated or on different networks.
+
+The process uses a Windows PowerShell script and both the source and target computers
+that execute the code are required to be installed with Windows PowerShell.
+
+The folder contents at source are archived and encrypted into a single file.  You 
+transfer the file to your target, where the content are unpacked using the decryption
+key. After archive contents are restored you can execute the reconcile function
+to veriy that the contents are transferred, unaltered.
+
+If you have access to both source and target folders, then you should consider
+using tools such as:
+
+* Microsoft ROBOCOPY
+* rsync
+
+Alternatively, you can use backup and restore utilities on the folder, and rely that 
+the contents are restored correctly.  If you want this to be secure, ensure
+the backup is encrypted.
+
+**Note**: If you require reconciliation (comparison) of files between the source
+and target, then you may be required to use additional software.  An example is
+JAM Software FileList.
+
+**Note**: Disk size utilities are not suitable for transferring/copying content
+
+## Background
+
+The script was born out a necessity to transfer a large volume of photographs
+from one server to another, where shared network drives was not a feasible
+solution.
+
+## Usage
+
+Packages source folder contents into a 7ZIP file, adding a reconciliation
+file to the 7ZIP file and then encrypting the contents.  Send
+
+* this script
+* the 7ZIP package file 
+* plus optional SecretFilename ( if using RecipientKeyName ) to the target or recipient.
+
+The source folder is not altered and only read rights are required. A log
+file is written at exceution to record activity.
+
+The SecretFileName can be sent via email, while the 7ZIP can go different routes
+due to possible size such as:
+
+* Cloud storage provider
+* HTTPS web file upload
+* SFTP transfer
+* USB stick
+
+At the target, unpack the contents to a folder and reconcile the results.  You
+will need write access on the target storage. A log file is written at exceution
+to record activity.
+
+Your bulk file transfer is encrypted in transit.  Note that if you use the
+SecretKey method the ecnrypted contents will only be as secure as the strength
+of your secret.
+
+You can use storage providers such as Dropbox, AWS S3, Google Drive, OneDrive or BackBlaze
+and your documents have additonal protection.
+
+A log file is produced on execution.  Repeated executions on the same day
+will add text content to the same log file.  The default log name takes the form:
+"ptr_files_yyyy-MM-dd.log"
+
+You will need to have installed the 7Zip4Powershell PowerShell cmdlet 
+before using the pack or unpack actions.  You can install the cmdlet
+by executing 
+.\ptrFiles.ps1 -Action install -Path ".\"

ptfrFiles/ptrFiles.ps1

@@ -46,11 +46,12 @@
 
  .Parameter Action
   Action to perform, which can be:
-  - Install
+  - Install             : Install 7Zip4PowerShell
-  - Pack
+  - Pack                : Archive the contents of a folder(s)
-  - Unpack
+  - Unpack              : Unpack the archive, but no reconfile is performed
-  - Reconcile
+  - Reconcile           : Reconcile the contents in the restored folder
-  - ReconcileFile
+  - ReconcileFile       : Generate reconfile file.  The pack process does this.
+  - ArchiveInformation  : Fetch archive information
 
  .Parameter Path
   The path to the files and folders to pack or the path to the unpack location. 
@@ -59,7 +60,13 @@
 
   When using the trailing * for names, the filtering is only applied to immediate
   folder names under the parent folder.  The filter does not cascade to lower folders.
-  
+
+  The Path can also be a file containing a list of paths, one per line.  To use a
+  list file, prefix the Path value with a "@" and name the file. Do not use a folder
+  for @ defined path.
+
+  A file (@ prefix) containing a list of paths cannot contain generic path names, that 
+  is paths with trailing wildcard of "*"
 
  .Parameter RecipientKeyName
   The recipient of the package which is used to find the appropriate
@@ -80,6 +87,8 @@
   uses a symmetric cryptographic key exchange which is less secure then the 
   RecipientKeyName approach.
 
+  Note: Currently the script doe snot user Secure Strings
+
  .Parameter ArchiveFileName
   The location and name of the 7ZIP file.  If not supplied a default 7ZIP file name
   will be generated in the current directory.
@@ -337,31 +346,33 @@ Param(
         Write-Host "Using @ file '$($folderName.Substring(1))'"
 
         Get-Content -Path $($folderName.Substring(1)) | ForEach-Object {
-            If (!(Test-Path -Path $_ )) {    
+            if ($_ -ne "") {
-                Write-Log "Folder/file '$($_)' does not exist"
+                If (!(Test-Path -Path $_ )) {    
-                Write-Host "Folder/file '$($_)' does not exist" -ForegroundColor Red
+                    Write-Log "Folder/file '$($_)' does not exist"
-            }
+                    Write-Host "Folder/file '$($_)' does not exist" -ForegroundColor Red
-            else {
+                }
-                Get-ChildItem $_ -Filter $fileFilter -Recurse | Where-Object {!$_.PSIsContainer} | ForEach-Object {
+                else {
+                    Get-ChildItem $_ -Filter $fileFilter -Recurse | Where-Object {!$_.PSIsContainer} | ForEach-Object {
 
-                    $totalFilecount = $totalFileCount + 1
+                        $totalFilecount = $totalFileCount + 1
-                    $totalFileSize = $totalFileSize + $_.Length 
+                        $totalFileSize = $totalFileSize + $_.Length 
-        
+            
-                    if (($totalFilecount % $messageFrequency) -eq 0) {            
+                        if (($totalFilecount % $messageFrequency) -eq 0) {            
-                        Write-Log "Read $totalFileCount files and size $(Get-ConvenientFileSize -Size $totalFileSize ).  Currently at folder '$($_.Directory)' " 
+                            Write-Log "Read $totalFileCount files and size $(Get-ConvenientFileSize -Size $totalFileSize ).  Currently at folder '$($_.Directory)' " 
-                        Write-Host "Read $totalFileCount files and size $(Get-ConvenientFileSize -Size $totalFileSize ).  Currently at folder '$($_.Directory)' " 
+                            Write-Host "Read $totalFileCount files and size $(Get-ConvenientFileSize -Size $totalFileSize ).  Currently at folder '$($_.Directory)' " 
+                        }
+            
+                        if ($ExcludeHash) {
+                            $sourceHash = ""
+                        } else {
+                            $sourceHash = (Get-FileHash -Path $_.FullName).Hash
+                        }
+                        $record = '"'+$_.FullName.Replace($rootFolderName, "")+'","'+$_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")+'"'
+                        $record = $record + ',"'+$_.CreationTime.ToString("yyyy-MM-ddTHH:mm:ss")+'","'+$_.LastAccessTime.ToString("yyyy-MM-ddTHH:mm:ss")+'"'
+                        $record = $record + ','+$_.Length+',"'+$sourceHash+'","'+ $_.Directory + '","' + $_.Name + '","' + $_.Attributes+'","'+$_.Extension+'"'
+                        Add-Content -Path  $reconcileFile  -Value $record
+                    
                     }
-        
-                    if ($ExcludeHash) {
-                        $sourceHash = ""
-                    } else {
-                        $sourceHash = (Get-FileHash -Path $_.FullName).Hash
-                    }
-                    $record = '"'+$_.FullName.Replace($rootFolderName, "")+'","'+$_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")+'"'
-                    $record = $record + ',"'+$_.CreationTime.ToString("yyyy-MM-ddTHH:mm:ss")+'","'+$_.LastAccessTime.ToString("yyyy-MM-ddTHH:mm:ss")+'"'
-                    $record = $record + ','+$_.Length+',"'+$sourceHash+'","'+ $_.Directory + '","' + $_.Name + '","' + $_.Attributes+'","'+$_.Extension+'"'
-                    Add-Content -Path  $reconcileFile  -Value $record
-                
                 }
             }
         }
@@ -458,7 +469,6 @@ Param(
     }
 
     Write-Log "Saving folders/files to archive file '$compressFile'"
-    Write-Log "Source folder is '$transferFolder'"
     Write-Host "Saving folders/files to archive file '$compressFile'"
 
     if ($reconcileFile -eq "")
@@ -473,6 +483,7 @@ Param(
 
     if ($transferFolder.EndsWith("*"))
     {
+        Write-Log "Archive primary folder is '$transferFolder'"
         $firstCompress = $true
 
         Get-ChildItem $transferFolder| ForEach-Object {
@@ -501,28 +512,30 @@ Param(
             Write-Host "Using @ file '$($transferFolder.Substring(1))'"
 
             Get-Content -Path $($transferFolder.Substring(1)) | ForEach-Object {
-                If (!(Test-Path -Path $_ )) {    
+                if ($_ -ne "") {
-                    Write-Log "Folder/file '$($_)' does not exist"
+                    If (!(Test-Path -Path $_ )) {    
-                    Write-Host "Folder/file '$($_)' does not exist" -ForegroundColor Red
+                        Write-Log "Folder/file '$($_)' does not exist"
-                }
+                        Write-Host "Folder/file '$($_)' does not exist" -ForegroundColor Red
-                else {
+                    }
-                    Write-Log "Archive folder '$($_)'"
+                    else {
-                    Write-Host "Archivefolder '$($_)'"
+                        Write-Log "Archive folder '$($_)'"
-                    if (Test-Files -FolderName $_ -FileFilter $fileFilter) {
+                        Write-Host "Archivefolder '$($_)'"
-                        try {
+                        if (Test-Files -FolderName $_ -FileFilter $fileFilter) {
-                            if ($firstCompress) {
+                            try {
-                                Compress-7Zip -Path $_ -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Filter $fileFilter   
+                                if ($firstCompress) {
-                            } else {
+                                    Compress-7Zip -Path $_ -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Filter $fileFilter   
-                                Compress-7Zip -Path $_ -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Filter $fileFilter -Append    
+                                } else {
+                                    Compress-7Zip -Path $_ -ArchiveFileName $compressFile -Format SevenZip -PreserveDirectoryRoot -Filter $fileFilter -Append    
+                                }
+                                $firstCompress = $false
+                            } catch {
+                                Write-Log "Compress error with file '$($_)'.  See any previous errors.  $Error"
+                                Write-Host "Compress error with file '$($_)'.  See any previous errors.  $Error" -ForegroundColor Red
                             }
-                            $firstCompress = $false
+                        } else {
-                        } catch {
+                            Write-Log "Empty folder '$($_.FullName)'"
-                            Write-Log "Compress error with file '$($_)'.  See any previous errors.  $Error"
+                            Write-Host "Empty folder '$($_.FullName)'"
-                            Write-Host "Compress error with file '$($_)'.  See any previous errors.  $Error" -ForegroundColor Red
                         }
-                    } else {
-                        Write-Log "Empty folder '$($_.FullName)'"
-                        Write-Host "Empty folder '$($_.FullName)'"
                     }
                 }
             }
@@ -590,7 +603,8 @@ function Invoke-Reconcile
 Param( 
     [Parameter(Mandatory)][String] $ReconcileFile,
     [Parameter(Mandatory)][String] $Folder,
-    [String] $TargetReconcileFile
+    [String] $TargetReconcileFile,
+    [Switch] $ExtendedCheck
 ) 
 
     if ($reconcileFile -eq "")
@@ -617,6 +631,7 @@ Param(
     $totalFileCount = 0
     $totalFileSize = 0
     $errorCount = 0
+    $missingFileCount = 0
     $missingHash = $false
 
     # For each entry in the reconcile file
@@ -629,17 +644,35 @@ Param(
                 $targetHash= (Get-FileHash -Path $restoreFileName).Hash
                 if ($_.Hash -ne $targetHash) {
                     $errorCount = $errorCount + 1
-                    Write-Log "Hash mismatch for file '$restoreFileName'"
+                    Write-Log "Hash mismatch for file '$restoreFileName' with target value $targetHash"
                 }
             } else {
                 $missingHash = $true
             }
-            if ((Get-Item -Path $restoreFileName).LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.LastWriteTime) {
+            if ((Get-Item -Path $restoreFileName).CreationTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.CreationTime) {
                 $errorCount = $errorCount + 1
-                Write-Log "Last write mismatch for file '$restoreFileName'"
+                Write-Log "Creation mismatch for file '$restoreFileName' with target value $((Get-Item -Path $restoreFileName).CreationTime.ToString("yyyy-MM-ddTHH:mm:ss"))"
             }
+            if ((Get-Item -Path $restoreFileName).Length -ne $_.Length) {
+                $errorCount = $errorCount + 1
+                Write-Log "Length mismatch for file '$restoreFileName' with target value $(Get-Item -Path $restoreFileName).Length)"
+            }
+
+            # Note that last / write access time is not checked by default as it will comonly be changed after restore
+            if ($extendedCheck) {
+                if ((Get-Item -Path $restoreFileName).LastAccessTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.LastAccessTime) {
+                    $errorCount = $errorCount + 1
+                    Write-Log "Last access mismatch for file '$restoreFileName' with target value $((Get-Item -Path $restoreFileName).LastAccessTime.ToString("yyyy-MM-ddTHH:mm:ss"))"
+                }
+                if ((Get-Item -Path $restoreFileName).LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss") -ne $_.LastWriteTime) {
+                    $errorCount = $errorCount + 1
+                    Write-Log "Last write mismatch for file '$restoreFileName' with target value $((Get-Item -Path $restoreFileName).LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss"))"
+                }
+            }
+
             $totalFileSize = $totalFileSize + (Get-Item -Path $restoreFileName).Length             
         } else {
+            $missingFileCount = $missingFileCount + 1
             $errorCount = $errorCount + 1
             Write-Log "Non existant target file '$restoreFileName'"
         }
@@ -648,17 +681,23 @@ Param(
     Write-Log "Total file storage size is $(Get-ConvenientFileSize -Size $totalFileSize ) ($totalFileSize)"
     Write-Host "Total file storage size is $(Get-ConvenientFileSize -Size $totalFileSize )"
 
-    Write-Log "Total file count is $totalFileCount with $errorCount errors"
     if ($missingHash)
     {
         Write-Log "Reconcile file had one or many or all blank hash entries"
         Write-Host "Reconcile file had one or many or all blank hash entries"  -ForegroundColor Yellow
     }
+
+    Write-Log "Total file count is $totalFileCount with $errorCount errors"
+    Write-Log "There are $missingFileCount missing files"
+
     if ($errorCount -gt 0) {
         Write-Host "Total file count is $totalFileCount with $errorCount errors" -ForegroundColor Red
     } else {
         Write-Host "Total file count is $totalFileCount with $errorCount errors"  -ForegroundColor Green
     }
+    if ($missingFileCount -gt 0) {
+        Write-Host "There are $missingFileCount missing files" -ForegroundColor Red
+    }
 }
 
 $dateTimeStart = Get-Date -f "yyyy-MM-dd HH:mm:ss"
@@ -770,17 +809,43 @@ if ($action -eq "Reconcile") {
     Invoke-Reconcile -ReconcileFile $localReconcileFile -Folder $path
 }
 
+if ($action -eq "ArchiveInformation") {
+    $actioned = $true
+    if (($RecipientKeyName -eq "") -and ($SecretKey -eq "")) {
+        Write-Log "Recipient Key Name or Secret Key required for 7Zip information" 
+        Write-Host "Recipient Key Name or Secret Key required for 7Zip information"  -ForegroundColor Red
+        Close-Log
+        return
+    } 
+    
+    if ($SecretKey -eq "") {
+        if ($secretFileName -eq "")
+        {
+            $secretFileName = $default_secretEncrypted
+        }
+        $secret = Unprotect-CmsMessage -To $recipientKeyName -Path $secretFileName
+    } else {
+        $secret = $SecretKey
+    }
+    Write-Log "Retrieving archive information"
+    Write-Host "Retrieving archive information"
+    
+    Get-7ZipInformation -ArchiveFileName $ArchiveFileName -Password $secret
+}
+
 
 if (!($actioned))
 {
     Write-Log "Unknown action '$action'.  No processing performed" 
     Write-Host "Unknown action '$action'.  No processing performed"  -ForegroundColor Red
     Write-Host "Recognised actions: "
-    Write-Host "    Pack          : Pack folder contents into secure 7Zip file"
+    Write-Host "    Pack                 : Pack folder contents into secure 7Zip file"
-    Write-Host "    Unpack        : Unpack folder contents from secure 7Zip file"
+    Write-Host "    Unpack               : Unpack folder contents from secure 7Zip file"
-    Write-Host "    Reconcile     : Reconcile files in unpack folder with list of packed files"
+    Write-Host "    Reconcile            : Reconcile files in unpack folder with list of packed files"
-    Write-Host "    ReconcileFile : Generate a reconcile file without packing"
+    Write-Host "    ReconcileFile        : Generate a reconcile file without packing"
-    Write-Host "    Install       : Install required packages"
+    Write-Host "    Install              : Install required packages"
+    Write-Host "    ArchiveInformation   : Fetch archive information from archive file"
+    
     Write-Host ""
     Write-Host "For help use command "
     Write-Host "    Get-Help .\ptrFiles.ps1"