Updating documentation

and adding metadata
pull/1/head
Tom Peltonen 2021-08-08 17:00:22 +10:00
parent a2c5f23852
commit 3a136d2a7a
6 changed files with 223 additions and 66 deletions

Binary file not shown.

View File

@ -30,10 +30,12 @@
#>
$global:default_reconcileFile = "##protect_transfer_reconcile_files##.csv"
$global:default_reconcileFile = "##peter_files##.csv"
$global:default_exifFile = "##peter_exif##.csv"
$global:default_metaFile = "##peter##.json"
$global:LogPathName = ""
$global:MetadataPathName = Join-Path -Path ".\" -ChildPath ".peter-metadata"
$global:Version = "0.0.1"
function Open-Log {
@ -301,7 +303,7 @@ Param(
}
$ExifFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_exifFile
Write-Log "Generating Exif file '$ExifFile'"
Set-Content -Path $ExifFile -Value '"FullName","Author","Title","Subject","Comments","DateTaken","ISO","FNumber"'
Set-Content -Encoding utf8 -Path $ExifFile -Value $(Set-ExifCsvHeader)
}
$totalFileCount = 0
@ -358,11 +360,8 @@ Param(
Add-Content -Path $ReconcileFile -Value $record
if ($IncludeExif) {
$exifData = Get-ImageFileContents -ImageFile $($_.FullName)
$exifRecord = '"' + $_.FullName + '","' + $exifData.Author + '","' + $exifData.Title + '","' + $exifData.Subject + '","' + $exifData.Comments + '"'
$exifRecord = $exifRecord + ',"' + $exifData.DateTaken + '","' + $exifData.Iso + '","' + $exifData.FNumber + '"'
Add-Content -Path $ExifFile -Value $exifRecord
$exifData = Get-ImageFileExif -ImageFile $($_.FullName)
Add-Content -Path $ExifFile -Value (Set-ExifCsvRecord -ExifData $exifData)
}
}
@ -397,11 +396,8 @@ Param(
Add-Content -Path $ReconcileFile -Value $record
if ($IncludeExif) {
$exifData = Get-ImageFileContents -ImageFile $($_.FullName)
$exifRecord = '"' + $_.FullName + '","' + $exifData.Author + '","' + $exifData.Title + '","' + $exifData.Subject + '","' + $exifData.Comments + '"'
$exifRecord = $exifRecord + ',"' + $exifData.DateTaken + '","' + $exifData.Iso + '","' + $exifData.FNumber + '"'
Add-Content -Path $ExifFile -Value $exifRecord
$exifData = Get-ImageFileExif -ImageFile $($_.FullName)
Add-Content -Path $ExifFile -Value (Set-ExifCsvRecord -ExifData $exifData )
}
}
@ -531,7 +527,7 @@ function Invoke-SinglePack
Once a reconcile is executed, you can delete this file from the
restored location.
The default name is "##protect_transfer_reconcile_files##.csv"
The default name is "##peter_files##.csv"
.Parameter SecretFile
The secret file name is used with RecipientKey to secure the
@ -751,11 +747,7 @@ Param(
$archiveInfo = Get-7ZipInformation -ArchiveFileName $ArchiveFile
[int] $archiveFileCount = $archiveInfo.FilesCount
if ($ExcludeHash) {
New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ExcludeHash -ProcessFileCount $archiveFileCount -IncludeExif
} else {
New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ProcessFileCount $archiveFileCount -IncludeExif
}
New-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ExcludeHash:$ExcludeHash -ProcessFileCount $archiveFileCount -IncludeExif:$IncludeExif
If (!(Test-Path -Path $ReconcileFile )) {
Write-Log "Reconcile file '$ReconcileFile' was not created. See any previous errors"
Write-Host "Reconcile file '$ReconcileFile' was not created. See any previous errors" -ForegroundColor Red
@ -763,6 +755,29 @@ Param(
return
}
# Write Json file as links
$jsonFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_metaFile
$jsonData = @{}
$dataItem = @{"SourceFolder"="$SourceFolder";"RecipientKey"="$RecipientKey";"ArchiveFile"="$ArchiveFile";"FileFilter"="$FileFilter";"SecretFile"="$SecretFile";"ExcludeHash"="$ExcludeHash";}
$jsonData.Add("Parameters",$dataItem)
$dataItem = @{"Name"="PeterDocs";"Author"="Meerkat@merebox.com";"Version"="$global:Version";}
$jsonData.Add("Software",$dataItem)
$items = New-Object System.Collections.ArrayList
$dataItem = @{"Reconcile"="$ReconcileFile";"Caption"="File listing of archive for reconciliation";}
$null = $items.Add($dataItem)
if ($IncludeExif) {
$ExifFile = Join-Path -Path $global:MetadataPathName -ChildPath $global:default_exifFile
$dataItem = @{"Exif"="$ExifFile";"Caption"="Exif information";}
$null = $items.Add($dataItem)
}
$jsonData.Add("Links",$items)
$jsonData | ConvertTo-Json -Depth 10 | Out-File $jsonFile
Write-Log "Add folder '$global:MetadataPathName' to file '$ArchiveFile'"
$fullMetadatName = (Get-Item $global:MetadataPathName).FullName
$fullZipName = (Get-Item $ArchiveFile).FullName
@ -1558,6 +1573,28 @@ Param(
return
}
# Check for metadata
$jsonFile = Join-Path -Path (Join-Path -Path $RestoreFolder -ChildPath $global:MetadataPathName ) -ChildPath $global:default_metaFile
Write-Host "Checking $jsonFile"
if (Test-FilesExist $jsonFile) {
$jsonData = Get-Content -Raw -Path $jsonFile | ConvertFrom-Json
$software = $jsonData.Software.Name
Write-Host "json: $software"
$jsonData.Links | ForEach-Object {
$_.PSObject.Properties | ForEach-object {
if ($_.Name -eq "Reconcile") {
if ($ReconcileFile -eq "") {
$ReconcileFile = Join-Path -Path $RestoreFolder -ChildPath $_.Value
Write-Log "Using metadata reconciliation file '$ReconcileFile'"
Write-Host "Using metadata reconciliation file '$ReconcileFile'"
}
}
}
}
}
if ($ReconcileFile -eq "")
{
$ReconcileFile = Join-Path -Path (Join-Path -Path $RestoreFolder -ChildPath $global:MetadataPathName) -ChildPath $default_reconcileFile

View File

@ -146,7 +146,7 @@ function MakeNumber {
}
function Get-ImageFileContents {
function Get-ImageFileExif {
param(
[Parameter(Mandatory)]
[String] $ImageFile
@ -289,4 +289,87 @@ function Get-ImageFileContents {
}
}
#Get-ImageFileContents -ImageFile "E:\tom\Projects\powershell\ptrFiles\assets\03.jpg"
function Set-ExifCsvHeader {
$ExifData = [PSCustomObject][ordered]@{
File = ""
DateTaken = ""
DateDigitized = ""
DateModified = ""
Author = ""
Title = ""
Subject = ""
Comments = ""
Keywords = ""
Artist = ""
Copyright = ""
Height = 0
Width = 0
PixelX = 0
PixelY = 0
ResolutionX = 0
ResolutionY = 0
CameraMaker = ""
CameraModel = ""
CameraLabel = ""
SoftwareVersion = ""
LatitudeRef = ""
Latitude = ""
LongitudeRef = ""
Longitude = ""
ExifVersion = ""
Flash = $false
Iso = 0
FocalLength = 0
ShutterSpeed = ""
Aperture = 0
FNumber = 0
}
$exifRecord = ''
$first = $true
$ExifData.PSObject.Properties | foreach-object {
if ($first) {
$exifRecord = '"' + $_.Name + '"'
} else{
$exifRecord = $exifRecord + ',"' + $_.Name + '"'
}
$first = $false
}
return $exifRecord
}
function Set-ExifCsvRecord {
param(
[Parameter(Mandatory)]
[PSCustomObject] $ExifData
)
$exifRecord = ''
$first = $true
$ExifData.PSObject.Properties | foreach-object {
if ($first) {
$exifRecord = '"' + $_.value + '"'
} else{
$exifRecord = $exifRecord + ',"' + $_.value + '"'
}
$first = $false
}
return $exifRecord
}

View File

@ -2,7 +2,7 @@
.Synopsis
Allows the secure transfer and reconciliation of a large number of files
PTRfile: Protect, Transfer, Reconcile files
PTR : Protect, Transfer, Reconcile files
.Description
Packages source folder contents into a 7ZIP file, adding a reconciliation
@ -43,14 +43,14 @@
.\ptrDocs.ps1 -Action install -Path ".\"
.Parameter Action
.Parameter Task
Action to perform, which can be:
- Pack : Archive the contents of a folder(s)
- Compress : Archive the contents of a folder(s)
- Put : Send the archive to AWS S3 or Backblaze
- Get : Receive the archive from AWS S3 or Backblaze
- Unpack : Unpack the archive, but no reconfile is performed
- Reconcile : Reconcile the contents in the restored folder
- ReconcileFile : Generate reconfile file. The pack process does this.
- Expand : Unpack the archive, but no reconfile is performed
- Compare : Reconcile the contents in the restored folder
- NewReconcile : Generate reconfile file. The pack process does this automatically.
- ArchiveInformation : Fetch archive information
.Parameter Path
@ -116,7 +116,7 @@
Once a reconcile is executed, you can delete this file from the
restored location.
The default name is "##protect_transfer_reconcile_files##.csv"
The default name is "##peter_files##.csv"
.Parameter SecretFile
The secret file name is used with RecipientKey to secure the
@ -140,6 +140,9 @@
generation to speed up the packaging. Excluding the hash does reduce
the functionality of the reconciliation at unpack.
.Parameter IncludeExif
Include Exif details into separate file for picture files.
.Parameter LogPath
The log folder where log files are written. If the folder does not
exist then it is created. You need write access rights to this location.
@ -170,41 +173,43 @@
.Example
# Pack and encrypt all files in folder ".\transferpack\" using a private-public key
# A file with the postifx ".key" is also generated alongside the 7ZIP file
.\ptrDocs.ps1 -Action pack -Path ".\transferpack\" -RecipientKey data@mycompany
.\PeterTask.ps1 -Task compress -Path ".\transferpack\" -RecipientKey data@mycompany
.Example
# Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a private-public key
# You will need the file "transfer_protect_yyyMMdd_hhmm.7z.key" to unpack the encrypted 7ZIP file
.\ptrDocs.ps1 -Action unpack -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKey data@mycompany
.\PeterTask.ps1 -Task expand -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKey data@mycompany
.Example
# Reconcile files in folder ".\targetdir"
.\ptrDocs.ps1 -Action reconcile -Path ".\targetdir"
.\PeterTask.ps1 -Task compare -Path ".\targetdir"
.Example
# Pack and encrypt all files in folder ".\transferpack\" using a password
.\ptrDocs.ps1 -Action pack -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds"
.\PeterTask.ps1 -Task compress -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds"
.Example
# Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a password
.\ptrDocs.ps1 -Action unpack -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds"
.\PeterTask.ps1 -Task expand -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds"
.Example
# Pack and encrypt all files in folder ".\transferpack\02*" where the folder name starts with "02" using a password
.\ptrDocs.ps1 -Action pack -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds"
.\PeterTask.ps1 -Task compress -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds"
#>
[CmdletBinding()]
param (
[Parameter(Mandatory)]
[Alias("Task")]
[String] $Action,
[ValidateSet("Compress","Expand","Compare","NewReconcile","Put","Get","ArchiveInformation")]
[Alias("Action")]
[String] $Task,
[Parameter(Mandatory)]
[Alias("Directory","DirectoryPath","Folder","FolderPath")]
[String] $Path,
[Alias("Recipient")]
[String] $RecipientKey,
[Alias("Password")]
@ -215,6 +220,7 @@ param (
[String] $RootFolder,
[Alias("Filter")]
[String] $FileFilter,
[String] $ReconcileFile,
@ -224,8 +230,10 @@ param (
[Alias("Profile", "Username")]
[String] $CloudProfile,
[Alias("Hash")]
[switch] $ExcludeHash,
[Alias("Exif")]
[switch] $IncludeExif,
[String] $LogPath = ""
@ -236,54 +244,42 @@ Import-Module .\PeterDocs
$actioned = $false
if ($action -eq "Pack") {
if ($task -eq "Compress") {
$actioned = $true
if ($ExcludeHash) {
Compress-Peter -ExcludeHash -SourceFolder $path -SecretKey $SecretKey -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -IncludeExif
} else {
Compress-Peter -SourceFolder $path -SecretKey $SecretKey -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -IncludeExif
}
Compress-Peter -SourceFolder $path -SecretKey $SecretKey -SecretFile $SecretFile -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter -LogPath $LogPath -ExcludeHash:$ExcludeHash -IncludeExif:$IncludeExif
}
if ($action -eq "Put") {
if ($task -eq "Put") {
$actioned = $true
Send-Peter -ArchiveFile $archiveFile -TargetPath $path -SecretFile $secretFile -TargetProfile $cloudProfile -LogPath $LogPath
}
if ($action -eq "Get") {
if ($task -eq "Get") {
$actioned = $true
Receive-Peter -ArchiveFile $archiveFile -SourcePath $path -SecretFile $secretFile -SourceProfile $cloudProfile -LogPath $LogPath
}
if ($action -eq "Unpack") {
if ($task -eq "Expand") {
$actioned = $true
Expand-Peter -RestoreFolder $path -SecretKey $secretKey -SecretFile $secretFile -ArchiveFile $ArchiveFile -LogPath $LogPath
}
if ($action -eq "ReconcileFile") {
if ($task -eq "NewReconcile") {
$actioned = $true
if ($ExcludeHash) {
New-PeterReconcile -ExcludeHash -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath
} else {
New-PeterReconcile -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath -IncludeExif
}
New-PeterReconcile -ReconcileFile $reconcileFile -SourceFolder $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter -LogPath LogPath -ExcludeHash:$ExcludeHash -IncludeExif:$IncludeExif
}
if ($action -eq "Reconcile") {
if ($task -eq "Compare") {
$actioned = $true
if ($ExcludeHash) {
Compare-Peter -ExcludeHash -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath
} else {
Compare-Peter -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath
}
Compare-Peter -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder -LogPath $LogPath -ExcludeHash:$ExcludeHash
}
if ($action -eq "ArchiveInformation") {
if ($task -eq "ArchiveInformation") {
$actioned = $true
if (($RecipientKey -eq "") -and ($SecretKey -eq "")) {
Write-Host "Recipient Key or Secret Key required for 7Zip information" -ForegroundColor Red
@ -306,14 +302,14 @@ Import-Module .\PeterDocs
if (!($actioned))
{
Write-Host "Unknown action '$action'. No processing performed" -ForegroundColor Red
Write-Host "Unknown action '$task'. No processing performed" -ForegroundColor Red
Write-Host "Recognised actions: "
Write-Host " Pack : Pack folder contents into secure 7Zip file"
Write-Host " Compress : Pack folder contents into secure 7Zip file"
Write-Host " Put : Put or send the archive file to remote destination"
Write-Host " Get : Get or fetch the archive from remote location"
Write-Host " Unpack : Unpack folder contents from secure 7Zip file"
Write-Host " Reconcile : Reconcile files in unpack folder with list of packed files"
Write-Host " ReconcileFile : Generate a reconcile file without packing"
Write-Host " Expand : Unpack folder contents from secure 7Zip file"
Write-Host " Compare : Reconcile files in unpack folder with list of packed files"
Write-Host " NewReconcile : Generate a reconcile file without packing"
Write-Host " ArchiveInformation : Fetch archive information from archive file"
Write-Host ""

38
QuickStart.md 100644
View File

@ -0,0 +1,38 @@
# PeterDocs - Quick Start
If you are in a hurry, comfortable to use ``PeterTask.ps1`` and want to accept the defaults
then this is what you need to do.
1. Download PowerShell file [PeterTask.ps1](https://raw.github.com/) to a lcoal directory.
2. Open a PowerShell terminal where the above file is stored locally and execute command
```powershell
Install-Module -Name PeterDocs -Scope CurrentUser
```
3. Run command below substituting the names as applicable
```powershell
.\PeterTask.ps1 -Task Compress -ArchiveFile .\myfiles.7z -Path <document path> -SecretKey <complex password>
```
4. Send the 7Zip file to where you wish to save or restore. _Hint:_ See [SendArchive](./Docs/SendArchive.md)
5. If you are restoring then run command below substituting the names as applicable
```powershell
.\PeterTask.ps1 -Task Expand -ArchiveFile .\myfiles.7z -Path <restore path> -SecretKey <complex password>
```
6. If you restored and want to reconcile the restore then run command below substituting the names as applicable
```powershell
.\PeterTask.ps1 -Task Compare -Path <restore path>
```
## Advanced security
If you are sending the documents and only want the recipient to be able to unpack the contents then
read the [Encryption](./Docs/Encryption.md) document.
Using **-RecipientKey** option in above commands secures the contents to only the recipient using
asymmetric encryption on an internal complex password.

View File

@ -32,10 +32,13 @@ JAM Software FileList.
## Background
The script was born out of necessity to transfer a large volume of photographs
from a media server to cloud storage. Commonly photographs are stored in many
from a media server to cloud storage for backup. Commonly photographs are stored in many
folders and can be large in number and size because of the increased
resolution of digital cameras.
The backup also required to be secure from accidental distribution. The backup is not secured
from accidental or malicious deletion, which are require different controls.
## Usage
Some basic commands in sequence are demonstrated below:
@ -78,7 +81,7 @@ due to possible size such as:
* USB stick
At the target, unpack the contents to a folder and reconcile the results. You
will need write access on the target storage. A log file is written at exceution
will need write access on the target storage. A log file is written at execution
to record activity.
Your bulk file transfer is encrypted in transit. Note that if you use the