Updating documentation

and adding metadata

PeterDocs/PeterDocs.psm1

@@ -30,10 +30,12 @@
 #>
 
 
-$global:default_reconcileFile = "##protect_transfer_reconcile_files##.csv"
+$global:default_reconcileFile = "##peter_files##.csv"
 $global:default_exifFile = "##peter_exif##.csv"
+$global:default_metaFile = "##peter##.json"
 $global:LogPathName = ""
 $global:MetadataPathName = Join-Path -Path ".\" -ChildPath ".peter-metadata"
+$global:Version = "0.0.1"
 
 function Open-Log {
     
@@ -301,7 +303,7 @@ Param(
         }
         $ExifFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_exifFile
         Write-Log "Generating Exif file '$ExifFile'"
-        Set-Content -Path $ExifFile  -Value '"FullName","Author","Title","Subject","Comments","DateTaken","ISO","FNumber"'
+        Set-Content  -Encoding utf8  -Path $ExifFile  -Value $(Set-ExifCsvHeader)
     }
 
     $totalFileCount = 0
@@ -358,11 +360,8 @@ Param(
                         Add-Content -Path  $ReconcileFile  -Value $record
 
                         if ($IncludeExif) {
-                            $exifData = Get-ImageFileContents -ImageFile $($_.FullName)
+                            $exifData = Get-ImageFileExif -ImageFile $($_.FullName)
-                            $exifRecord = '"' + $_.FullName + '","' + $exifData.Author + '","' + $exifData.Title + '","' + $exifData.Subject + '","' + $exifData.Comments + '"'
+                            Add-Content -Path $ExifFile  -Value (Set-ExifCsvRecord -ExifData $exifData)
-                            $exifRecord = $exifRecord + ',"' +  $exifData.DateTaken + '","' + $exifData.Iso + '","' + $exifData.FNumber + '"'
-                
-                            Add-Content -Path $ExifFile  -Value $exifRecord
                         }
                     
                     }
@@ -397,11 +396,8 @@ Param(
             Add-Content -Path  $ReconcileFile  -Value $record
 
             if ($IncludeExif) {
-                $exifData = Get-ImageFileContents -ImageFile $($_.FullName)
+                $exifData = Get-ImageFileExif -ImageFile $($_.FullName)
-                $exifRecord = '"' + $_.FullName + '","' + $exifData.Author + '","' + $exifData.Title + '","' + $exifData.Subject + '","' + $exifData.Comments + '"'
+                Add-Content -Path $ExifFile  -Value (Set-ExifCsvRecord -ExifData $exifData )
-                $exifRecord = $exifRecord + ',"' +  $exifData.DateTaken + '","' + $exifData.Iso + '","' + $exifData.FNumber + '"'
-
-                Add-Content -Path $ExifFile  -Value $exifRecord
             }
 
         }
@@ -531,7 +527,7 @@ function Invoke-SinglePack
   Once a reconcile is executed, you can delete this file from the 
   restored location.
 
-  The default name is "##protect_transfer_reconcile_files##.csv"
+  The default name is "##peter_files##.csv"
 
  .Parameter SecretFile
   The secret file name is used with RecipientKey to secure the
@@ -751,11 +747,7 @@ Param(
     $archiveInfo = Get-7ZipInformation -ArchiveFileName $ArchiveFile
     [int] $archiveFileCount = $archiveInfo.FilesCount
 
-    if ($ExcludeHash) {
+    New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ExcludeHash:$ExcludeHash -ProcessFileCount $archiveFileCount -IncludeExif:$IncludeExif
-        New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ExcludeHash -ProcessFileCount $archiveFileCount -IncludeExif
-    } else {
-        New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ProcessFileCount $archiveFileCount -IncludeExif
-    }
     If (!(Test-Path -Path $ReconcileFile )) {    
         Write-Log "Reconcile file '$ReconcileFile' was not created.  See any previous errors"
         Write-Host "Reconcile file '$ReconcileFile' was not created.  See any previous errors" -ForegroundColor Red
@@ -763,11 +755,34 @@ Param(
         return
     }
 
+    # Write Json file as links
+    $jsonFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_metaFile
+    $jsonData = @{}
+
+    $dataItem = @{"SourceFolder"="$SourceFolder";"RecipientKey"="$RecipientKey";"ArchiveFile"="$ArchiveFile";"FileFilter"="$FileFilter";"SecretFile"="$SecretFile";"ExcludeHash"="$ExcludeHash";}
+    $jsonData.Add("Parameters",$dataItem)
+
+    $dataItem = @{"Name"="PeterDocs";"Author"="Meerkat@merebox.com";"Version"="$global:Version";}
+    $jsonData.Add("Software",$dataItem)
+
+    $items = New-Object System.Collections.ArrayList
+    $dataItem = @{"Reconcile"="$ReconcileFile";"Caption"="File listing of archive for reconciliation";}
+    $null = $items.Add($dataItem)
+    
+    if ($IncludeExif) {
+        $ExifFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_exifFile
+        $dataItem = @{"Exif"="$ExifFile";"Caption"="Exif information";}
+        $null = $items.Add($dataItem)
+    }
+
+    $jsonData.Add("Links",$items)
+    $jsonData | ConvertTo-Json -Depth 10 | Out-File $jsonFile
+
     Write-Log "Add folder '$global:MetadataPathName' to file '$ArchiveFile'"
     $fullMetadatName = (Get-Item $global:MetadataPathName).FullName
     $fullZipName = (Get-Item $ArchiveFile).FullName
     Compress-7Zip -Path $fullMetadatName -ArchiveFileName $fullZipName -PreserveDirectoryRoot -Format SevenZip -Append -Password $secret -EncryptFilenames
-    #Remove-Item $fullMetadatName -Recurse
+#    Remove-Item $fullMetadatName -Recurse
 
     Write-Log "Archive file '$ArchiveFile' created from folder '$SourceFolder'"
     Write-Host "Archive file '$ArchiveFile' created from folder '$SourceFolder'"  -ForegroundColor Green
@@ -1558,6 +1573,28 @@ Param(
         return
     }
 
+
+    # Check for metadata
+    $jsonFile = Join-Path -Path (Join-Path -Path $RestoreFolder -ChildPath $global:MetadataPathName ) -ChildPath $global:default_metaFile
+    Write-Host "Checking $jsonFile"
+    if (Test-FilesExist $jsonFile) {
+        $jsonData = Get-Content -Raw -Path $jsonFile | ConvertFrom-Json
+
+        $software = $jsonData.Software.Name
+        Write-Host "json: $software"
+        $jsonData.Links | ForEach-Object {
+            $_.PSObject.Properties | ForEach-object {
+                if ($_.Name -eq "Reconcile") {
+                    if ($ReconcileFile -eq "") {
+                        $ReconcileFile = Join-Path -Path $RestoreFolder -ChildPath $_.Value
+                        Write-Log "Using metadata reconciliation file '$ReconcileFile'"
+                        Write-Host "Using metadata reconciliation file '$ReconcileFile'" 
+                    }
+                }
+            }
+        }
+    }
+
     if ($ReconcileFile -eq "")
     {
         $ReconcileFile = Join-Path -Path (Join-Path -Path $RestoreFolder -ChildPath $global:MetadataPathName) -ChildPath $default_reconcileFile

PeterDocs/PeterExif.ps1

@@ -146,7 +146,7 @@ function MakeNumber {
 }
 
 
-function Get-ImageFileContents {
+function Get-ImageFileExif {
     param(
         [Parameter(Mandatory)]
         [String] $ImageFile
@@ -289,4 +289,87 @@ function Get-ImageFileContents {
     }
 }
 
-#Get-ImageFileContents -ImageFile "E:\tom\Projects\powershell\ptrFiles\assets\03.jpg"
+
+
+function Set-ExifCsvHeader {
+
+    
+    $ExifData = [PSCustomObject][ordered]@{
+        File = ""
+
+        DateTaken = ""
+        DateDigitized = ""
+        DateModified = ""
+
+        Author = ""
+        Title = ""
+        Subject = ""
+        Comments = ""
+        Keywords = ""
+
+        Artist = ""
+        Copyright = ""
+
+        Height = 0
+        Width = 0
+        PixelX = 0
+        PixelY = 0
+        ResolutionX = 0
+        ResolutionY = 0
+
+        CameraMaker = ""
+        CameraModel = ""
+        CameraLabel = ""
+        SoftwareVersion = ""
+
+        LatitudeRef = ""
+        Latitude = ""
+        LongitudeRef = ""
+        Longitude = ""
+
+        ExifVersion = ""
+
+        Flash = $false
+        Iso = 0
+        FocalLength = 0
+        ShutterSpeed = ""
+        Aperture = 0
+        FNumber = 0
+
+    }
+
+    $exifRecord = ''
+    $first = $true
+    $ExifData.PSObject.Properties | foreach-object {
+        if ($first) {
+            $exifRecord = '"' + $_.Name + '"'
+        } else{
+            $exifRecord = $exifRecord + ',"' + $_.Name + '"'
+        }
+        $first = $false
+    }
+
+    return $exifRecord
+
+}
+
+function Set-ExifCsvRecord {
+    param(
+        [Parameter(Mandatory)]
+        [PSCustomObject] $ExifData
+    )
+
+    $exifRecord = ''
+    $first = $true
+    $ExifData.PSObject.Properties | foreach-object {
+        if ($first) {
+            $exifRecord = '"' + $_.value + '"'
+        } else{
+            $exifRecord = $exifRecord + ',"' + $_.value + '"'
+        }
+        $first = $false
+    }
+
+    return $exifRecord
+
+}

PeterTask.ps1

@@ -2,7 +2,7 @@
  .Synopsis
    Allows the secure transfer and reconciliation of a large number of files
 
-   PTRfile: Protect, Transfer, Reconcile files
+   PTR : Protect, Transfer, Reconcile files
 
  .Description
    Packages source folder contents into a 7ZIP file, adding a reconciliation 
@@ -43,14 +43,14 @@
    .\ptrDocs.ps1 -Action install -Path ".\" 
 
 
- .Parameter Action
+ .Parameter Task
   Action to perform, which can be:
-  - Pack                : Archive the contents of a folder(s)
+  - Compress            : Archive the contents of a folder(s)
   - Put                 : Send the archive to AWS S3 or Backblaze
   - Get                 : Receive the archive from AWS S3 or Backblaze
-  - Unpack              : Unpack the archive, but no reconfile is performed
+  - Expand              : Unpack the archive, but no reconfile is performed
-  - Reconcile           : Reconcile the contents in the restored folder
+  - Compare             : Reconcile the contents in the restored folder
-  - ReconcileFile       : Generate reconfile file.  The pack process does this.
+  - NewReconcile        : Generate reconfile file.  The pack process does this automatically.
   - ArchiveInformation  : Fetch archive information
 
  .Parameter Path
@@ -116,7 +116,7 @@
   Once a reconcile is executed, you can delete this file from the 
   restored location.
 
-  The default name is "##protect_transfer_reconcile_files##.csv"
+  The default name is "##peter_files##.csv"
 
  .Parameter SecretFile
   The secret file name is used with RecipientKey to secure the
@@ -140,6 +140,9 @@
   generation to speed up the packaging.  Excluding the hash does reduce 
   the functionality of the reconciliation at unpack.
 
+ .Parameter IncludeExif
+  Include Exif details into separate file for picture files.
+
  .Parameter LogPath
   The log folder where log files are written.  If the folder does not
   exist then it is created.  You need write access rights to this location.
@@ -170,41 +173,43 @@
  .Example
    # Pack and encrypt all files in folder ".\transferpack\" using a private-public key
    # A file with the postifx ".key" is also generated alongside the 7ZIP file
-   .\ptrDocs.ps1 -Action pack -Path ".\transferpack\" -RecipientKey data@mycompany
+   .\PeterTask.ps1 -Task compress -Path ".\transferpack\" -RecipientKey data@mycompany
  
  .Example
    # Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a private-public key
    # You will need the file "transfer_protect_yyyMMdd_hhmm.7z.key" to unpack the encrypted 7ZIP file
-   .\ptrDocs.ps1 -Action unpack -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKey data@mycompany
+   .\PeterTask.ps1 -Task expand -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKey data@mycompany
  
  .Example
    # Reconcile files in folder ".\targetdir"
-   .\ptrDocs.ps1 -Action reconcile -Path ".\targetdir" 
+   .\PeterTask.ps1 -Task compare -Path ".\targetdir" 
 
  .Example
    # Pack and encrypt all files in folder ".\transferpack\" using a password
-   .\ptrDocs.ps1 -Action pack -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds"
+   .\PeterTask.ps1 -Task compress -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds"
  
  .Example
    # Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a password
-   .\ptrDocs.ps1 -Action unpack -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds"
+   .\PeterTask.ps1 -Task expand -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds"
 
  .Example
    # Pack and encrypt all files in folder ".\transferpack\02*" where the folder name starts with "02" using a password
-   .\ptrDocs.ps1 -Action pack -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds"
+   .\PeterTask.ps1 -Task compress -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds"
 
 #>
 
 [CmdletBinding()]
 param (
     [Parameter(Mandatory)]
-    [Alias("Task")]
+    [ValidateSet("Compress","Expand","Compare","NewReconcile","Put","Get","ArchiveInformation")]
-    [String] $Action, 
+    [Alias("Action")]
+    [String] $Task, 
 
     [Parameter(Mandatory)]
     [Alias("Directory","DirectoryPath","Folder","FolderPath")]
     [String] $Path, 
 
+    [Alias("Recipient")]
     [String] $RecipientKey,
 
     [Alias("Password")]
@@ -215,6 +220,7 @@ param (
 
     [String] $RootFolder,
 
+    [Alias("Filter")]
     [String] $FileFilter,
 
     [String] $ReconcileFile, 
@@ -224,8 +230,10 @@ param (
     [Alias("Profile", "Username")]
     [String] $CloudProfile,
 
+    [Alias("Hash")]
     [switch] $ExcludeHash,
 
+    [Alias("Exif")]
     [switch] $IncludeExif,
 
     [String] $LogPath = ""
@@ -236,54 +244,42 @@ Import-Module .\PeterDocs
 
     $actioned = $false
 
-    if ($action -eq "Pack") {
+    if ($task -eq "Compress") {
         $actioned = $true
-        if ($ExcludeHash) {
+        Compress-Peter -SourceFolder $path -SecretKey $SecretKey  -SecretFile $SecretFile -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -ExcludeHash:$ExcludeHash -IncludeExif:$IncludeExif
-          Compress-Peter -ExcludeHash -SourceFolder $path -SecretKey $SecretKey -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -IncludeExif
-        } else {
-          Compress-Peter -SourceFolder $path -SecretKey $SecretKey -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -IncludeExif
-        }
     }
 
 
-    if ($action -eq "Put") {
+    if ($task -eq "Put") {
         $actioned = $true       
         Send-Peter -ArchiveFile $archiveFile -TargetPath $path -SecretFile $secretFile -TargetProfile $cloudProfile -LogPath $LogPath
     }
 
 
-    if ($action -eq "Get") {
+    if ($task -eq "Get") {
         $actioned = $true
         Receive-Peter -ArchiveFile $archiveFile -SourcePath $path -SecretFile $secretFile -SourceProfile $cloudProfile -LogPath $LogPath
     }
 
 
-    if ($action -eq "Unpack") {
+    if ($task -eq "Expand") {
         $actioned = $true
-        Expand-Peter -RestoreFolder $path -SecretKey $secretKey -SecretFile $secretFile -ArchiveFile $ArchiveFile -LogPath $LogPath
+        Expand-Peter -RestoreFolder $path -SecretKey $secretKey -SecretFile $secretFile -ArchiveFile $ArchiveFile -LogPath $LogPath 
     }
 
 
-    if ($action -eq "ReconcileFile") {
+    if ($task -eq "NewReconcile") {
         $actioned = $true
-        if ($ExcludeHash) {
+        New-PeterReconcile -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath  -ExcludeHash:$ExcludeHash -IncludeExif:$IncludeExif
-          New-PeterReconcile -ExcludeHash -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath
-        } else {
-          New-PeterReconcile -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath -IncludeExif
-        }
     }
 
 
-    if ($action -eq "Reconcile") {
+    if ($task -eq "Compare") {
         $actioned = $true
-        if ($ExcludeHash) {
+        Compare-Peter -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath -ExcludeHash:$ExcludeHash 
-          Compare-Peter -ExcludeHash -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath
-        } else {
-          Compare-Peter -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath
-        }
     }
 
-    if ($action -eq "ArchiveInformation") {
+    if ($task -eq "ArchiveInformation") {
         $actioned = $true
         if (($RecipientKey -eq "") -and ($SecretKey -eq "")) {
             Write-Host "Recipient Key or Secret Key required for 7Zip information"  -ForegroundColor Red
@@ -306,14 +302,14 @@ Import-Module .\PeterDocs
 
     if (!($actioned))
     {
-        Write-Host "Unknown action '$action'.  No processing performed"  -ForegroundColor Red
+        Write-Host "Unknown action '$task'.  No processing performed"  -ForegroundColor Red
         Write-Host "Recognised actions: "
-        Write-Host "    Pack                 : Pack folder contents into secure 7Zip file"
+        Write-Host "    Compress             : Pack folder contents into secure 7Zip file"
         Write-Host "    Put                  : Put or send the archive file to remote destination"
         Write-Host "    Get                  : Get or fetch the archive from remote location"
-        Write-Host "    Unpack               : Unpack folder contents from secure 7Zip file"
+        Write-Host "    Expand               : Unpack folder contents from secure 7Zip file"
-        Write-Host "    Reconcile            : Reconcile files in unpack folder with list of packed files"
+        Write-Host "    Compare              : Reconcile files in unpack folder with list of packed files"
-        Write-Host "    ReconcileFile        : Generate a reconcile file without packing"
+        Write-Host "    NewReconcile         : Generate a reconcile file without packing"
         Write-Host "    ArchiveInformation   : Fetch archive information from archive file"
         
         Write-Host ""

QuickStart.md

@@ -0,0 +1,38 @@
+# PeterDocs - Quick Start
+
+If you are in a hurry, comfortable to use ``PeterTask.ps1`` and want to accept the defaults
+then this is what you need to do.
+
+1. Download PowerShell file [PeterTask.ps1](https://raw.github.com/) to a lcoal directory.
+2. Open a PowerShell terminal where the above file is stored locally and execute command
+
+    ```powershell
+        Install-Module -Name PeterDocs  -Scope CurrentUser    
+    ```
+
+3. Run command below substituting the names as applicable
+
+    ```powershell
+        .\PeterTask.ps1 -Task Compress -ArchiveFile .\myfiles.7z  -Path <document path> -SecretKey <complex password>
+    ```
+
+4. Send the 7Zip file to where you wish to save or restore. _Hint:_ See [SendArchive](./Docs/SendArchive.md)
+5. If you are restoring then run command below substituting the names as applicable
+
+    ```powershell
+        .\PeterTask.ps1 -Task Expand -ArchiveFile .\myfiles.7z  -Path <restore path> -SecretKey <complex password>
+    ```
+
+6. If you restored and want to reconcile the restore then run command below substituting the names as applicable
+
+    ```powershell
+        .\PeterTask.ps1 -Task Compare -Path <restore path> 
+    ```
+
+## Advanced security
+
+If you are sending the documents and only want the recipient to be able to unpack the contents then
+read the [Encryption](./Docs/Encryption.md) document.
+
+Using **-RecipientKey** option in above commands secures the contents to only the recipient using
+asymmetric encryption on an internal complex password.

README.md

@@ -32,10 +32,13 @@ JAM Software FileList.
 ## Background
 
 The script was born out of necessity to transfer a large volume of photographs
-from a media server to cloud storage.  Commonly photographs are stored in many
+from a media server to cloud storage for backup.  Commonly photographs are stored in many
 folders and can be large in number and size because of the increased
 resolution of digital cameras.
 
+The backup also required to be secure from accidental distribution.  The backup is not secured
+from accidental or malicious deletion, which are require different controls.
+
 ## Usage
 
 Some basic commands in sequence are demonstrated below:
@@ -78,7 +81,7 @@ due to possible size such as:
 * USB stick
 
 At the target, unpack the contents to a folder and reconcile the results.  You
-will need write access on the target storage. A log file is written at exceution
+will need write access on the target storage. A log file is written at execution
 to record activity.
 
 Your bulk file transfer is encrypted in transit.  Note that if you use the