From 3a136d2a7a2cc51e7686309b500a948c4103b345 Mon Sep 17 00:00:00 2001 From: Tom Peltonen Date: Sun, 8 Aug 2021 17:00:22 +1000 Subject: [PATCH] Updating documentation and adding metadata --- PeterDocs/PeterDocs.psd1 | Bin 8582 -> 8574 bytes PeterDocs/PeterDocs.psm1 | 75 ++++++++++++++++++++++++--------- PeterDocs/PeterExif.ps1 | 87 ++++++++++++++++++++++++++++++++++++++- PeterTask.ps1 | 82 ++++++++++++++++++------------------ QuickStart.md | 38 +++++++++++++++++ README.md | 7 +++- 6 files changed, 223 insertions(+), 66 deletions(-) create mode 100644 QuickStart.md diff --git a/PeterDocs/PeterDocs.psd1 b/PeterDocs/PeterDocs.psd1 index 51db17c4980792f8ede1d095dcc7cb7affbbf90d..17afca179e72efc434ef8344c32103e60fd5afac 100644 GIT binary patch delta 20 bcmZp3{^zuzL4@6vp@JckA#HQN$afw9Pd^7b delta 28 gcmez8)aJaQL4@C#A)g_Sp@bn7Ocrl$7x~Tu0Eb}+`~Uy| diff --git a/PeterDocs/PeterDocs.psm1 b/PeterDocs/PeterDocs.psm1 index a8c193b..0fbb886 100644 --- a/PeterDocs/PeterDocs.psm1 +++ b/PeterDocs/PeterDocs.psm1 @@ -30,10 +30,12 @@ #> -$global:default_reconcileFile = "##protect_transfer_reconcile_files##.csv" +$global:default_reconcileFile = "##peter_files##.csv" $global:default_exifFile = "##peter_exif##.csv" +$global:default_metaFile = "##peter##.json" $global:LogPathName = "" $global:MetadataPathName = Join-Path -Path ".\" -ChildPath ".peter-metadata" +$global:Version = "0.0.1" function Open-Log { @@ -301,7 +303,7 @@ Param( } $ExifFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_exifFile Write-Log "Generating Exif file '$ExifFile'" - Set-Content -Path $ExifFile -Value '"FullName","Author","Title","Subject","Comments","DateTaken","ISO","FNumber"' + Set-Content -Encoding utf8 -Path $ExifFile -Value $(Set-ExifCsvHeader) } $totalFileCount = 0 @@ -358,11 +360,8 @@ Param( Add-Content -Path $ReconcileFile -Value $record if ($IncludeExif) { - $exifData = Get-ImageFileContents -ImageFile $($_.FullName) - $exifRecord = '"' + $_.FullName + '","' + $exifData.Author + '","' + $exifData.Title + '","' + $exifData.Subject + '","' + $exifData.Comments + '"' - $exifRecord = $exifRecord + ',"' + $exifData.DateTaken + '","' + $exifData.Iso + '","' + $exifData.FNumber + '"' - - Add-Content -Path $ExifFile -Value $exifRecord + $exifData = Get-ImageFileExif -ImageFile $($_.FullName) + Add-Content -Path $ExifFile -Value (Set-ExifCsvRecord -ExifData $exifData) } } @@ -397,11 +396,8 @@ Param( Add-Content -Path $ReconcileFile -Value $record if ($IncludeExif) { - $exifData = Get-ImageFileContents -ImageFile $($_.FullName) - $exifRecord = '"' + $_.FullName + '","' + $exifData.Author + '","' + $exifData.Title + '","' + $exifData.Subject + '","' + $exifData.Comments + '"' - $exifRecord = $exifRecord + ',"' + $exifData.DateTaken + '","' + $exifData.Iso + '","' + $exifData.FNumber + '"' - - Add-Content -Path $ExifFile -Value $exifRecord + $exifData = Get-ImageFileExif -ImageFile $($_.FullName) + Add-Content -Path $ExifFile -Value (Set-ExifCsvRecord -ExifData $exifData ) } } @@ -531,7 +527,7 @@ function Invoke-SinglePack Once a reconcile is executed, you can delete this file from the restored location. - The default name is "##protect_transfer_reconcile_files##.csv" + The default name is "##peter_files##.csv" .Parameter SecretFile The secret file name is used with RecipientKey to secure the @@ -751,11 +747,7 @@ Param( $archiveInfo = Get-7ZipInformation -ArchiveFileName $ArchiveFile [int] $archiveFileCount = $archiveInfo.FilesCount - if ($ExcludeHash) { - New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ExcludeHash -ProcessFileCount $archiveFileCount -IncludeExif - } else { - New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ProcessFileCount $archiveFileCount -IncludeExif - } + New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ExcludeHash:$ExcludeHash -ProcessFileCount $archiveFileCount -IncludeExif:$IncludeExif If (!(Test-Path -Path $ReconcileFile )) { Write-Log "Reconcile file '$ReconcileFile' was not created. See any previous errors" Write-Host "Reconcile file '$ReconcileFile' was not created. See any previous errors" -ForegroundColor Red @@ -763,11 +755,34 @@ Param( return } + # Write Json file as links + $jsonFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_metaFile + $jsonData = @{} + + $dataItem = @{"SourceFolder"="$SourceFolder";"RecipientKey"="$RecipientKey";"ArchiveFile"="$ArchiveFile";"FileFilter"="$FileFilter";"SecretFile"="$SecretFile";"ExcludeHash"="$ExcludeHash";} + $jsonData.Add("Parameters",$dataItem) + + $dataItem = @{"Name"="PeterDocs";"Author"="Meerkat@merebox.com";"Version"="$global:Version";} + $jsonData.Add("Software",$dataItem) + + $items = New-Object System.Collections.ArrayList + $dataItem = @{"Reconcile"="$ReconcileFile";"Caption"="File listing of archive for reconciliation";} + $null = $items.Add($dataItem) + + if ($IncludeExif) { + $ExifFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_exifFile + $dataItem = @{"Exif"="$ExifFile";"Caption"="Exif information";} + $null = $items.Add($dataItem) + } + + $jsonData.Add("Links",$items) + $jsonData | ConvertTo-Json -Depth 10 | Out-File $jsonFile + Write-Log "Add folder '$global:MetadataPathName' to file '$ArchiveFile'" $fullMetadatName = (Get-Item $global:MetadataPathName).FullName $fullZipName = (Get-Item $ArchiveFile).FullName Compress-7Zip -Path $fullMetadatName -ArchiveFileName $fullZipName -PreserveDirectoryRoot -Format SevenZip -Append -Password $secret -EncryptFilenames - #Remove-Item $fullMetadatName -Recurse +# Remove-Item $fullMetadatName -Recurse Write-Log "Archive file '$ArchiveFile' created from folder '$SourceFolder'" Write-Host "Archive file '$ArchiveFile' created from folder '$SourceFolder'" -ForegroundColor Green @@ -1558,6 +1573,28 @@ Param( return } + + # Check for metadata + $jsonFile = Join-Path -Path (Join-Path -Path $RestoreFolder -ChildPath $global:MetadataPathName ) -ChildPath $global:default_metaFile + Write-Host "Checking $jsonFile" + if (Test-FilesExist $jsonFile) { + $jsonData = Get-Content -Raw -Path $jsonFile | ConvertFrom-Json + + $software = $jsonData.Software.Name + Write-Host "json: $software" + $jsonData.Links | ForEach-Object { + $_.PSObject.Properties | ForEach-object { + if ($_.Name -eq "Reconcile") { + if ($ReconcileFile -eq "") { + $ReconcileFile = Join-Path -Path $RestoreFolder -ChildPath $_.Value + Write-Log "Using metadata reconciliation file '$ReconcileFile'" + Write-Host "Using metadata reconciliation file '$ReconcileFile'" + } + } + } + } + } + if ($ReconcileFile -eq "") { $ReconcileFile = Join-Path -Path (Join-Path -Path $RestoreFolder -ChildPath $global:MetadataPathName) -ChildPath $default_reconcileFile diff --git a/PeterDocs/PeterExif.ps1 b/PeterDocs/PeterExif.ps1 index aaac7c9..deb95c0 100644 --- a/PeterDocs/PeterExif.ps1 +++ b/PeterDocs/PeterExif.ps1 @@ -146,7 +146,7 @@ function MakeNumber { } -function Get-ImageFileContents { +function Get-ImageFileExif { param( [Parameter(Mandatory)] [String] $ImageFile @@ -289,4 +289,87 @@ function Get-ImageFileContents { } } -#Get-ImageFileContents -ImageFile "E:\tom\Projects\powershell\ptrFiles\assets\03.jpg" + + +function Set-ExifCsvHeader { + + + $ExifData = [PSCustomObject][ordered]@{ + File = "" + + DateTaken = "" + DateDigitized = "" + DateModified = "" + + Author = "" + Title = "" + Subject = "" + Comments = "" + Keywords = "" + + Artist = "" + Copyright = "" + + Height = 0 + Width = 0 + PixelX = 0 + PixelY = 0 + ResolutionX = 0 + ResolutionY = 0 + + CameraMaker = "" + CameraModel = "" + CameraLabel = "" + SoftwareVersion = "" + + LatitudeRef = "" + Latitude = "" + LongitudeRef = "" + Longitude = "" + + ExifVersion = "" + + Flash = $false + Iso = 0 + FocalLength = 0 + ShutterSpeed = "" + Aperture = 0 + FNumber = 0 + + } + + $exifRecord = '' + $first = $true + $ExifData.PSObject.Properties | foreach-object { + if ($first) { + $exifRecord = '"' + $_.Name + '"' + } else{ + $exifRecord = $exifRecord + ',"' + $_.Name + '"' + } + $first = $false + } + + return $exifRecord + +} + +function Set-ExifCsvRecord { + param( + [Parameter(Mandatory)] + [PSCustomObject] $ExifData + ) + + $exifRecord = '' + $first = $true + $ExifData.PSObject.Properties | foreach-object { + if ($first) { + $exifRecord = '"' + $_.value + '"' + } else{ + $exifRecord = $exifRecord + ',"' + $_.value + '"' + } + $first = $false + } + + return $exifRecord + +} \ No newline at end of file diff --git a/PeterTask.ps1 b/PeterTask.ps1 index 29ae730..b34a9e3 100644 --- a/PeterTask.ps1 +++ b/PeterTask.ps1 @@ -2,7 +2,7 @@ .Synopsis Allows the secure transfer and reconciliation of a large number of files - PTRfile: Protect, Transfer, Reconcile files + PTR : Protect, Transfer, Reconcile files .Description Packages source folder contents into a 7ZIP file, adding a reconciliation @@ -43,14 +43,14 @@ .\ptrDocs.ps1 -Action install -Path ".\" - .Parameter Action + .Parameter Task Action to perform, which can be: - - Pack : Archive the contents of a folder(s) + - Compress : Archive the contents of a folder(s) - Put : Send the archive to AWS S3 or Backblaze - Get : Receive the archive from AWS S3 or Backblaze - - Unpack : Unpack the archive, but no reconfile is performed - - Reconcile : Reconcile the contents in the restored folder - - ReconcileFile : Generate reconfile file. The pack process does this. + - Expand : Unpack the archive, but no reconfile is performed + - Compare : Reconcile the contents in the restored folder + - NewReconcile : Generate reconfile file. The pack process does this automatically. - ArchiveInformation : Fetch archive information .Parameter Path @@ -116,7 +116,7 @@ Once a reconcile is executed, you can delete this file from the restored location. - The default name is "##protect_transfer_reconcile_files##.csv" + The default name is "##peter_files##.csv" .Parameter SecretFile The secret file name is used with RecipientKey to secure the @@ -140,6 +140,9 @@ generation to speed up the packaging. Excluding the hash does reduce the functionality of the reconciliation at unpack. + .Parameter IncludeExif + Include Exif details into separate file for picture files. + .Parameter LogPath The log folder where log files are written. If the folder does not exist then it is created. You need write access rights to this location. @@ -170,41 +173,43 @@ .Example # Pack and encrypt all files in folder ".\transferpack\" using a private-public key # A file with the postifx ".key" is also generated alongside the 7ZIP file - .\ptrDocs.ps1 -Action pack -Path ".\transferpack\" -RecipientKey data@mycompany + .\PeterTask.ps1 -Task compress -Path ".\transferpack\" -RecipientKey data@mycompany .Example # Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a private-public key # You will need the file "transfer_protect_yyyMMdd_hhmm.7z.key" to unpack the encrypted 7ZIP file - .\ptrDocs.ps1 -Action unpack -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKey data@mycompany + .\PeterTask.ps1 -Task expand -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKey data@mycompany .Example # Reconcile files in folder ".\targetdir" - .\ptrDocs.ps1 -Action reconcile -Path ".\targetdir" + .\PeterTask.ps1 -Task compare -Path ".\targetdir" .Example # Pack and encrypt all files in folder ".\transferpack\" using a password - .\ptrDocs.ps1 -Action pack -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds" + .\PeterTask.ps1 -Task compress -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds" .Example # Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a password - .\ptrDocs.ps1 -Action unpack -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds" + .\PeterTask.ps1 -Task expand -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds" .Example # Pack and encrypt all files in folder ".\transferpack\02*" where the folder name starts with "02" using a password - .\ptrDocs.ps1 -Action pack -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds" + .\PeterTask.ps1 -Task compress -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds" #> [CmdletBinding()] param ( [Parameter(Mandatory)] - [Alias("Task")] - [String] $Action, + [ValidateSet("Compress","Expand","Compare","NewReconcile","Put","Get","ArchiveInformation")] + [Alias("Action")] + [String] $Task, [Parameter(Mandatory)] [Alias("Directory","DirectoryPath","Folder","FolderPath")] [String] $Path, + [Alias("Recipient")] [String] $RecipientKey, [Alias("Password")] @@ -215,6 +220,7 @@ param ( [String] $RootFolder, + [Alias("Filter")] [String] $FileFilter, [String] $ReconcileFile, @@ -224,8 +230,10 @@ param ( [Alias("Profile", "Username")] [String] $CloudProfile, + [Alias("Hash")] [switch] $ExcludeHash, + [Alias("Exif")] [switch] $IncludeExif, [String] $LogPath = "" @@ -236,54 +244,42 @@ Import-Module .\PeterDocs $actioned = $false - if ($action -eq "Pack") { + if ($task -eq "Compress") { $actioned = $true - if ($ExcludeHash) { - Compress-Peter -ExcludeHash -SourceFolder $path -SecretKey $SecretKey -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -IncludeExif - } else { - Compress-Peter -SourceFolder $path -SecretKey $SecretKey -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -IncludeExif - } + Compress-Peter -SourceFolder $path -SecretKey $SecretKey -SecretFile $SecretFile -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -ExcludeHash:$ExcludeHash -IncludeExif:$IncludeExif } - if ($action -eq "Put") { + if ($task -eq "Put") { $actioned = $true Send-Peter -ArchiveFile $archiveFile -TargetPath $path -SecretFile $secretFile -TargetProfile $cloudProfile -LogPath $LogPath } - if ($action -eq "Get") { + if ($task -eq "Get") { $actioned = $true Receive-Peter -ArchiveFile $archiveFile -SourcePath $path -SecretFile $secretFile -SourceProfile $cloudProfile -LogPath $LogPath } - if ($action -eq "Unpack") { + if ($task -eq "Expand") { $actioned = $true - Expand-Peter -RestoreFolder $path -SecretKey $secretKey -SecretFile $secretFile -ArchiveFile $ArchiveFile -LogPath $LogPath + Expand-Peter -RestoreFolder $path -SecretKey $secretKey -SecretFile $secretFile -ArchiveFile $ArchiveFile -LogPath $LogPath } - if ($action -eq "ReconcileFile") { + if ($task -eq "NewReconcile") { $actioned = $true - if ($ExcludeHash) { - New-PeterReconcile -ExcludeHash -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath - } else { - New-PeterReconcile -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath -IncludeExif - } + New-PeterReconcile -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath -ExcludeHash:$ExcludeHash -IncludeExif:$IncludeExif } - if ($action -eq "Reconcile") { + if ($task -eq "Compare") { $actioned = $true - if ($ExcludeHash) { - Compare-Peter -ExcludeHash -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath - } else { - Compare-Peter -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath - } + Compare-Peter -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath -ExcludeHash:$ExcludeHash } - if ($action -eq "ArchiveInformation") { + if ($task -eq "ArchiveInformation") { $actioned = $true if (($RecipientKey -eq "") -and ($SecretKey -eq "")) { Write-Host "Recipient Key or Secret Key required for 7Zip information" -ForegroundColor Red @@ -306,14 +302,14 @@ Import-Module .\PeterDocs if (!($actioned)) { - Write-Host "Unknown action '$action'. No processing performed" -ForegroundColor Red + Write-Host "Unknown action '$task'. No processing performed" -ForegroundColor Red Write-Host "Recognised actions: " - Write-Host " Pack : Pack folder contents into secure 7Zip file" + Write-Host " Compress : Pack folder contents into secure 7Zip file" Write-Host " Put : Put or send the archive file to remote destination" Write-Host " Get : Get or fetch the archive from remote location" - Write-Host " Unpack : Unpack folder contents from secure 7Zip file" - Write-Host " Reconcile : Reconcile files in unpack folder with list of packed files" - Write-Host " ReconcileFile : Generate a reconcile file without packing" + Write-Host " Expand : Unpack folder contents from secure 7Zip file" + Write-Host " Compare : Reconcile files in unpack folder with list of packed files" + Write-Host " NewReconcile : Generate a reconcile file without packing" Write-Host " ArchiveInformation : Fetch archive information from archive file" Write-Host "" diff --git a/QuickStart.md b/QuickStart.md new file mode 100644 index 0000000..98d046a --- /dev/null +++ b/QuickStart.md @@ -0,0 +1,38 @@ +# PeterDocs - Quick Start + +If you are in a hurry, comfortable to use ``PeterTask.ps1`` and want to accept the defaults +then this is what you need to do. + +1. Download PowerShell file [PeterTask.ps1](https://raw.github.com/) to a lcoal directory. +2. Open a PowerShell terminal where the above file is stored locally and execute command + + ```powershell + Install-Module -Name PeterDocs -Scope CurrentUser + ``` + +3. Run command below substituting the names as applicable + + ```powershell + .\PeterTask.ps1 -Task Compress -ArchiveFile .\myfiles.7z -Path -SecretKey + ``` + +4. Send the 7Zip file to where you wish to save or restore. _Hint:_ See [SendArchive](./Docs/SendArchive.md) +5. If you are restoring then run command below substituting the names as applicable + + ```powershell + .\PeterTask.ps1 -Task Expand -ArchiveFile .\myfiles.7z -Path -SecretKey + ``` + +6. If you restored and want to reconcile the restore then run command below substituting the names as applicable + + ```powershell + .\PeterTask.ps1 -Task Compare -Path + ``` + +## Advanced security + +If you are sending the documents and only want the recipient to be able to unpack the contents then +read the [Encryption](./Docs/Encryption.md) document. + +Using **-RecipientKey** option in above commands secures the contents to only the recipient using +asymmetric encryption on an internal complex password. diff --git a/README.md b/README.md index acf5a5e..cdd4f4d 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,13 @@ JAM Software FileList. ## Background The script was born out of necessity to transfer a large volume of photographs -from a media server to cloud storage. Commonly photographs are stored in many +from a media server to cloud storage for backup. Commonly photographs are stored in many folders and can be large in number and size because of the increased resolution of digital cameras. +The backup also required to be secure from accidental distribution. The backup is not secured +from accidental or malicious deletion, which are require different controls. + ## Usage Some basic commands in sequence are demonstrated below: @@ -78,7 +81,7 @@ due to possible size such as: * USB stick At the target, unpack the contents to a folder and reconcile the results. You -will need write access on the target storage. A log file is written at exceution +will need write access on the target storage. A log file is written at execution to record activity. Your bulk file transfer is encrypted in transit. Note that if you use the