diff --git a/Compress.md b/Compress.md new file mode 100644 index 0000000..bb91a91 --- /dev/null +++ b/Compress.md @@ -0,0 +1,46 @@ +# Compress and Protect + +## Source Documents + +## Why + +## When + +## How + +To perform the create the archive file you execute the ```Compress-Peter``` function. + +```powershell +Compress-Peter + -SourceFolder + -RecipientKey + -SecretKey + -ArchiveFile + -ReconcileFile + -FileFilter + -SecretFile + -ExcludeHash + -RootFolder + -LogPath +``` + +The function requires a ```SourceFolder```. + +Either a ```RecipientKey``` or ```SecretKey``` is required. + +If no ```ArchiveFile``` name is specified a default name is used. + +You can ignore the remaining parameters if you are happy with the defaults. + +## What + +The ```Compress-Peter``` compressess the contet of the ```SourceFolder``` and saves the result +as the encrypted ```ArchiveFile```. The archive file also contains the reconciliation file +so that the recipient of the archive is able to reconcile the restore at the remote location. + +If a ```RecipientKey``` is used then an extra file (```SecretFile``) is also created. Do not +loose this file as without it you cannot decrypt the archive contents. + +## Send Usage + +Please read next the documentation on [sending the archive](SendArchive.md) diff --git a/Expand.md b/Expand.md new file mode 100644 index 0000000..2d445f7 --- /dev/null +++ b/Expand.md @@ -0,0 +1,44 @@ +# Expand Archive + +## Why + +The objective is to restore or clone the documents in a new location. + +## When + +The documents can be restored/cloned after the archive file and key file +(if applicable) are received at the new location. + +## How + +To perform the expand and restore/clone you execute the ```Expand-Peter``` function. + +```powershell +Expand-Peter + -ArchiveFile + -RestoreFolder + -RecipientKey + -SecretKey + -SecretFile + -LogPath +``` + +If you encrypted the archive file with a ```RecipientKey``` then you will need +the private key of the recipient and the ".key" file. You can specify the +```SecretFile``` if it is not the default name of the archive file followed +by the extension ".key" + +You cannot decrypt the archive file if you do not have the private key or the +".key" file. + +To expand the archive you will need write access to the ```RestoreFolder``` location. + +## What + +The ```Expand-Peter``` decrypts the archive file and expands the contents into +the specified restore folder. It does not peform a reconciliation which is the +next step. + +## Reconcile Usage + +Please read next the documentation on [reconciling the archive](Reconcile.md) diff --git a/Install.md b/Install.md new file mode 100644 index 0000000..36ebc3f --- /dev/null +++ b/Install.md @@ -0,0 +1,47 @@ +# Install + +PeterDocs is a module that can be donwloaded or installed from +[PowerShell Gallery](https://xx.com/) + +## Pre-requisites + +PowerShell must be installed before you can use the PeterDocs module. + +## Automated install + +A generic script is available to allow you to install the required +modules. The same script can be used to exceute as a sample to +execute the actual packing, unpacking and reconciliation. + +You can get the generic script from [Github as ptrDocs.ps1](https://raw.githubusercontent.com/meerkat-manor/ptrFiles/main/ptrDocs.ps1) + +After downloading the file, execute the script as follows to install the modules + +```powershell +.\ptrDocs.ps1 -Action install -Path .\ +``` + +## Manual install + +Execute the following commands to install the module under the current user + +```powershell + Install-Module -Name 7Zip4Powershell -Scope CurrentUser + Install-Module -Name AWS.Tools.Installer -Scope CurrentUser + Install-Module -Name AWS.Tools.S3 -Scope CurrentUser + Install-Module -Name Meerkat.PeterDocs -Scope CurrentUser +``` + +Execute the following commands to install the module for all users. You will +need administrator rights. + +```powershell + Install-Module -Name 7Zip4Powershell -Scope AllUsers + Install-Module -Name AWS.Tools.Installer -Scope AllUsers + Install-Module -Name AWS.Tools.S3 -Scope AllUsers + Install-Module -Name Meerkat.PeterDocs -Scope AllUsers +``` + +## Compress Usage + +Please read next the documentation on [creating an archive file](Compress.md) diff --git a/PeterDocs/PeterB2.ps1 b/PeterDocs/PeterB2.psm1 similarity index 58% rename from PeterDocs/PeterB2.ps1 rename to PeterDocs/PeterB2.psm1 index e613444..521c3bb 100644 --- a/PeterDocs/PeterB2.ps1 +++ b/PeterDocs/PeterB2.psm1 @@ -1,9 +1,4 @@ -$default_reconcileFile = "##protect_transfer_reconcile_files##.csv" -$default_profile = "default" -$default_archiveFile = ".\ptr_file_##date##.7z" - - function Get-B2ApiToken { Param @@ -141,7 +136,7 @@ function Get-B2UploadUri { } -function Invoke-B2SUpload { +function Send-B2Upload { Param ( [Parameter(Mandatory)] [ValidateNotNull()] [ValidateNotNullOrEmpty()] [String] $BucketHost, @@ -197,7 +192,7 @@ function Invoke-B2SUpload { -function Invoke-B2SDownload { +function Receive-B2Download { Param ( [Parameter(Mandatory)] [ValidateNotNull()] [ValidateNotNullOrEmpty()] [String] $BucketHost, @@ -233,134 +228,3 @@ function Invoke-B2SDownload { } - -# Compress / Package - -<# - .Synopsis - Packs a source folder(s) into an encrypted 7ZIP archive file - that can be securely transported to a remote lcoation or - even used as a secure permmanent backup. - - PeterDocs : Protect, Transfer, Reconcile Document Files - - .Description - Packages source folder contents into a 7ZIP file, adding a reconciliation - file to the 7ZIP file and then encrypting the contents. The source folder - is not altered and only read rights are required. A log file is written - at exceution to record activity. - - - .Parameter SourceFolder - The path to the files and folders to pack. - The path name can include a trailing * as a wildcard to only include a subset of - directories. - - When using the trailing * for names, the filtering is only applied to immediate - folder names under the parent folder. The filter does not cascade to lower folders. - - The path can be a local drive, mapped network drive or a network shared folder - location such as \\MediaStor\MyLibrary. - - The source folder parameter can also be a file containing a list of paths, one per line. - To use a list file, prefix the source folder value with a "@" and name the file. - Do not use a folder for @ defined path. - - A file (@ prefix) containing a list of paths cannot contain generic path names, that - is paths with trailing wildcard of "*" - - .Parameter RecipientKey - The recipient of the package which is used to find the appropriate - certificate for encrypting with the public key. Either the RecipientKeyName - or the SecretKey is required for packing or unpacking the 7ZIP file. - Using the RecipientKeyName is the most secure transfer option as a - asymmetric cryptographic key is used that can only be decrypted by the - holder of the private key. - - If you are using the RecipientKeyName, then the 7ZIP file contents can only - be unzipped by the holder of the private key and the SecretFileName file. - If you don't have the private, which you should not unless you are sending - to yourself, then you cannot unpack the 7ZIP file. - - .Parameter SecretKey - A tradiitional secret to encrypt or decrypt the 7ZIP package. Either the RecipientKeyName - or the SecretKey is required for packing or unpacking the 7ZIP file. This method - uses a symmetric cryptographic key exchange which is less secure then the - RecipientKeyName approach. - - Note: Currently the script doe snot user Secure Strings - - .Parameter ArchiveFile - The location and name of the 7ZIP file. If not supplied a default 7ZIP file name - will be generated in the current directory for the pack action. - - The default name will take the form ".\transfer_protect_yyyyMMdd_hhmm.7z" - - For unpack actions, the archive file name parameter is mandatory. - - .Parameter RootFolder - The root folder, which should be used if using wildcard (*) for the - path. A guess will be made as to value if not supplied, which will - work in many circumstances. - - .Parameter FileFilter - A filter on file names. This does not filter directories. - An example to only include JPEG file is "*.jpg". You can also - filter on picture file names starting with "IMG*.jpg" - - .Parameter ReconcileFile - The name of the reconfile file name to generate during pack or use - during unpack. This is a file name without path. If no value is - supplied, then a default name is generated. - The reconcile file is included into the root of the 7ZIP file. - Once a reconcile is executed, you can delete this file from the - restored location. - - The default name is "##protect_transfer_reconcile_files##.csv" - - .Parameter SecretFile - The secret file name is used with RecipientKey to secure the - internally generated password for the 7ZIP file. When unpacking the - 7ZIP file you will need access to this file if RecipientKey - was used. If not supplied a default name is used. This file is - encrypted with RecipientKey. - - The default name is the archive file name with postfix ".key" - - .Parameter ExcludeHash - Exclude the file hash from the reconcile file. As producing a file - hash takes compute cycles during pack, you can select to bypass this - generation to speed up the packaging. Excluding the hash does reduce - the functionality of the reconciliation at unpack. - - .Parameter LogPath - The log folder where log files are written. If the folder does not - exist then it is created. You need write access rights to this location. - - .Notes - This script has been written to use the 7ZIP function as it is open source - and provides a secure encryption mechanism, plus portability on Windows, - Linux and MacOS. - - It is also beneficial that 7ZIP has efficient compression algorithms. - - Compressing and packing a large data set can take significant time and also - require storage space. The script does not check if you have sufficient - free storage to package the source contents into a single 7ZIP file. It is your - responsibility to ensure sufficient storage space exists. - - If you need to copy files from one directory to another accessible directory from - your Windows desktop, you might consider using ROBOCOPY. If the target directory - is not accessible and you want to reconcile, then this tool is appropriate. - - The following environment variables are supported: - - PETERDOCS_RECIPIENTKEY - - - .Example - # Pack and encrypt all files in folder ".\transferpack\" using a private-public key - # A file with the postifx ".key" is also generated alongside the 7ZIP file - Invoke-Pack -SourceFolder ".\transferpack\" -RecipientKeyName data@mycompany - -#> - diff --git a/PeterDocs/PeterDocs.psd1 b/PeterDocs/PeterDocs.psd1 index b5c8a3a..8173f0c 100644 Binary files a/PeterDocs/PeterDocs.psd1 and b/PeterDocs/PeterDocs.psd1 differ diff --git a/PeterDocs/PeterDocs.ps1 b/PeterDocs/PeterDocs.psm1 similarity index 95% rename from PeterDocs/PeterDocs.ps1 rename to PeterDocs/PeterDocs.psm1 index 2705588..18b6e52 100644 --- a/PeterDocs/PeterDocs.ps1 +++ b/PeterDocs/PeterDocs.psm1 @@ -30,6 +30,10 @@ #> +$global:default_reconcileFile = "##protect_transfer_reconcile_files##.csv" +$global:LogPathName = "" + + function Open-Log { $dateTimeStart = Get-Date -f "yyyy-MM-dd HH:mm:ss" @@ -46,15 +50,15 @@ function Write-Log { $date = Get-Date -f "yyyy-MM-dd" - if (($null -eq $LogPathName) -or ($LogPathName -eq "")) + if (($null -eq $global:LogPathName) -or ($global:LogPathName -eq "")) { - $LogPathName = Join-Path -Path ".\" -ChildPath "Logs" + $global:LogPathName = Join-Path -Path ".\" -ChildPath "Logs" } $logName = $(Get-SoftwareName) + "_$date.log" - $sFullPath = Join-Path -Path $LogPathName -ChildPath $logName + $sFullPath = Join-Path -Path $global:LogPathName -ChildPath $logName - if (!(Test-Path -Path $LogPathName)) { - $null = New-Item -Path $LogPathName -ItemType Directory + if (!(Test-Path -Path $global:LogPathName)) { + $null = New-Item -Path $global:LogPathName -ItemType Directory } if (!(Test-Path -Path $sFullPath)) { @@ -226,11 +230,11 @@ function Get-ConvenientFileSize .Example # Create a reconcile file for folder "C:\sourcefiles\" - Set-Reconcile -SourceFolder "C:\sourcefiles\" -ReconcileFile ".\myreconcile.csv" + Build-PeterReconcile -SourceFolder "C:\sourcefiles\" -ReconcileFile ".\myreconcile.csv" #> -function Set-Reconcile +function Build-PeterReconcile { Param( [Parameter(Mandatory)][String] $SourceFolder, @@ -246,13 +250,13 @@ Param( if (($null -ne $LogPath) -and ($LogPath -ne "")) { - $LogPathName = $LogPath + $global:LogPathName = $LogPath } if ($Feedback) { Open-Log - Write-Log "Function 'Set-Reconcile' parameters follow" + Write-Log "Function 'Build-PeterReconcile' parameters follow" Write-Log "Parameter: SourceFolder Value: $SourceFolder " Write-Log "Parameter: ReconcileFile Value: $ReconcileFile " Write-Log "Parameter: RootFolder Value: $RootFolder " @@ -552,11 +556,11 @@ function Invoke-SinglePack # Pack and encrypt all files in folder ".\transferpack\" using a private-public key # A default archive named file is created which includes a date and time in the name. # A file with the postifx ".key" is also generated alongside the 7ZIP file - Invoke-Pack -SourceFolder ".\transferpack\" -RecipientKeyName data@mycompany + Compress-Peter -SourceFolder ".\transferpack\" -RecipientKeyName data@mycompany #> -function Invoke-Pack +function Compress-Peter { Param( [Parameter(Mandatory)][String] $SourceFolder, @@ -574,12 +578,12 @@ Param( if (($null -ne $LogPath) -and ($LogPath -ne "")) { - $LogPathName = $LogPath + $global:LogPathName = $LogPath } Open-Log - Write-Log "Function 'Invoke-Pack' parameters follow" + Write-Log "Function 'Compress-Peter' parameters follow" Write-Log "Parameter: SourceFolder Value: $SourceFolder " Write-Log "Parameter: RecipientKey Value: $RecipientKey " if ($null -eq $SecretKey) { @@ -723,9 +727,9 @@ Param( [int] $archiveFileCount = $archiveInfo.FilesCount if ($ExcludeHash) { - Set-Reconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ExcludeHash -ProcessFileCount $archiveFileCount + Build-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ExcludeHash -ProcessFileCount $archiveFileCount } else { - Set-Reconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ProcessFileCount $archiveFileCount + Build-PeterReconcile -ReconcileFile $ReconcileFile -SourceFolder $SourceFolder -FileFilter $FileFilter -RootFolder $rootFolder -ProcessFileCount $archiveFileCount } If (!(Test-Path -Path $ReconcileFile )) { Write-Log "Reconcile file '$ReconcileFile' was not created. See any previous errors" @@ -831,11 +835,11 @@ Param( .Example # # - Invoke-PutArchive -ArchiveFile "mybackup.7z" -TargetPath + Send-Peter -ArchiveFile "mybackup.7z" -TargetPath #> -function Invoke-PutArchive +function Send-Peter { Param( [Parameter(Mandatory)][String] $ArchiveFile, @@ -849,12 +853,12 @@ Param( if (($null -ne $LogPath) -and ($LogPath -ne "")) { - $LogPathName = $LogPath + $global:LogPathName = $LogPath } Open-Log - Write-Log "Function 'Invoke-PutArchive' parameters follow" + Write-Log "Function 'Send-Peter' parameters follow" Write-Log "Parameter: ArchiveFile Value: $ArchiveFile " Write-Log "Parameter: TargetPath Value: $TargetPath " Write-Log "Parameter: SecretFile Value: $SecretFile " @@ -981,13 +985,13 @@ Param( $targetObject = $TargetPath.Substring($offset) Write-Log "Transferring '$ArchiveFile' file to host '$bucketHost' folder '$targetObject'" Write-Host "Transferring '$ArchiveFile' file to host '$bucketHost' folder '$targetObject'" - $b2Upload = Invoke-B2SUpload -BucketHost $b2UploadUri.bucketId -TargetPath $targetObject -FileName $ArchiveFile -ApiUri $b2UploadUri.uploadUri -ApiToken $b2UploadUri.Token + $b2Upload = Send-B2Upload -BucketHost $b2UploadUri.bucketId -TargetPath $targetObject -FileName $ArchiveFile -ApiUri $b2UploadUri.uploadUri -ApiToken $b2UploadUri.Token Write-Log "Upload: $b2Upload" if (Test-Path -Path $SecretFile) { $targetObject = $TargetPath.Substring($offset) + ".key" Write-Log "Transferring '$SecretFile' file to host '$bucketHost' folder '$targetObject'" Write-Host "Transferring '$SecretFile' file to host '$bucketHost' folder '$targetObject'" - $b2Upload = Invoke-B2SUpload -BucketHost $b2UploadUri.bucketId -TargetPath $targetObject -FileName $SecretFile -ApiUri $b2UploadUri.uploadUri -ApiToken $b2UploadUri.Token + $b2Upload = Send-B2Upload -BucketHost $b2UploadUri.bucketId -TargetPath $targetObject -FileName $SecretFile -ApiUri $b2UploadUri.uploadUri -ApiToken $b2UploadUri.Token Write-Log "Upload: $b2Upload" } $targetObject = $TargetPath.Substring($offset) @@ -1099,11 +1103,11 @@ Param( .Example # # - Invoke-PutArchive -ArchiveFile "mybackup.7z" -TargetPath + Receive-Peter -ArchiveFile "mybackup.7z" -TargetPath #> -function Invoke-GetArchive +function Receive-Peter { Param( [Parameter(Mandatory)][String] $SourcePath, @@ -1117,12 +1121,12 @@ Param( if (($null -ne $LogPath) -and ($LogPath -ne "")) { - $LogPathName = $LogPath + $global:LogPathName = $LogPath } Open-Log - Write-Log "Function 'Invoke-PutArchive' parameters follow" + Write-Log "Function 'Receive-Peter' parameters follow" Write-Log "Parameter: SourcePath Value: $SourcePath " Write-Log "Parameter: ArchiveFile Value: $ArchiveFile " Write-Log "Parameter: SecretFile Value: $SecretFile " @@ -1234,7 +1238,7 @@ Param( $sourceObject = $SourcePath.Substring($offset) Write-Log "Fetching '$ArchiveFile' file from host '$bucketHost' folder '$sourceObject'" Write-Host "Fetching '$ArchiveFile' file from host '$bucketHost' folder '$sourceObject'" - Invoke-B2SDownload -BucketHost $bucketHost -SourcePath $sourceObject -FileName $ArchiveFile -ApiDownloadUri $b2ApiToken.DownloadUri -ApiToken $b2ApiToken.Token + Receive-B2Download -BucketHost $bucketHost -SourcePath $sourceObject -FileName $ArchiveFile -ApiDownloadUri $b2ApiToken.DownloadUri -ApiToken $b2ApiToken.Token if (!(Test-Path -Path $ArchiveFile)) { Write-Log "Archive file '$sourceObject' not found." Write-Host "Archive file '$sourceObject' not found." -ForegroundColor Red @@ -1243,7 +1247,7 @@ Param( $secretFile = $ArchiveFile + ".key" Write-Log "Fetching '$secretFile' file from host '$bucketHost' folder '$sourceObject'" Write-Host "Fetching '$secretFile' file from host '$bucketHost' folder '$sourceObject'" - Invoke-B2SDownload -BucketHost $bucketHost -SourcePath $sourceObject -FileName $secretFile -ApiDownloadUri $b2ApiToken.DownloadUri -ApiToken $b2ApiToken.Token + Receive-B2Download -BucketHost $bucketHost -SourcePath $sourceObject -FileName $secretFile -ApiDownloadUri $b2ApiToken.DownloadUri -ApiToken $b2ApiToken.Token if (!(Test-Path -Path $secretFile)) { Write-Log "Secret file '$sourceObject' not found. Required if you are using recipient keys" Write-Host "Secret file '$sourceObject' not found. Required if you are using recipient keys" @@ -1350,16 +1354,16 @@ Param( # Unpack all the files in the archive file "myarchive.7z" into folder # ".\retsoredpack\" using a private-public key as decrypt and # checking for default file "myarchive.7z.key" - Invoke-Unpack -ArchiveFile "myarchive.7z" -RestoreFolder ".\restorepack\" -RecipientKey data@mycompany + Expand-Peter -ArchiveFile "myarchive.7z" -RestoreFolder ".\restorepack\" -RecipientKey data@mycompany .Example # Unpack all the files in the archive file "myarchive.7z" into folder # ".\restorepack\" using a secret of "longAndComplex9!key" - Invoke-Unpack -ArchiveFile "myarchive.7z" -RestoreFolder ".\restorepack\" -SecretKey "longAndComplex9!key" + Expand-Peter -ArchiveFile "myarchive.7z" -RestoreFolder ".\restorepack\" -SecretKey "longAndComplex9!key" #> -function Invoke-Unpack +function Expand-Peter { Param( [Parameter(Mandatory)][String] $ArchiveFile, @@ -1372,12 +1376,12 @@ Param( if (($null -ne $LogPath) -and ($LogPath -ne "")) { - $LogPathName = $LogPath + $global:LogPathName = $LogPath } Open-Log - Write-Log "Function 'Invoke-Unpack' parameters follow" + Write-Log "Function 'Expand-Peter' parameters follow" Write-Log "Parameter: ArchiveFile Value: $ArchiveFile " Write-Log "Parameter: RestoreFolder Value: $RestoreFolder " Write-Log "Parameter: RecipientKey Value: $RecipientKey " @@ -1490,15 +1494,15 @@ Param( .Example # Reconcile folder ".\restorefolder\" using default reconcile file - Invoke-Reconcile -RestoreFolder ".\transferfolder\" + Compare-Peter -RestoreFolder ".\transferfolder\" .Example # Reconcile folder ".\restorefolder\" using the reconcile # file located at "C:\reconcileme.csv" - Invoke-Reconcile -RestoreFolder ".\transferfolder\" -ReconcileFile "C:\reconcileme.csv" + Compare-Peter -RestoreFolder ".\transferfolder\" -ReconcileFile "C:\reconcileme.csv" #> -function Invoke-Reconcile +function Compare-Peter { Param( [Parameter(Mandatory)][String] $RestoreFolder, @@ -1510,12 +1514,12 @@ Param( if (($null -ne $LogPath) -and ($LogPath -ne "")) { - $LogPathName = $LogPath + $global:LogPathName = $LogPath } Open-Log - Write-Log "Function 'Invoke-Reconcile' parameters follow" + Write-Log "Function 'Compare-Peter' parameters follow" Write-Log "Parameter: RestoreFolder Value: $RestoreFolder " Write-Log "Parameter: ReconcileFile Value: $ReconcileFile " Write-Log "Parameter: RootFolder Value: $RootFolder " @@ -1645,10 +1649,7 @@ Param( Close-Log } - -$default_reconcileFile = "##protect_transfer_reconcile_files##.csv" -$LogPathName = "" $getEnvName = $(Get-SoftwareName) + "_LOGPATH" if ([System.Environment]::GetEnvironmentVariable($getEnvName) -ne "" -and $null -ne [System.Environment]::GetEnvironmentVariable($getEnvName)) { - $LogPathName = [System.Environment]::GetEnvironmentVariable($getEnvName) + $global:LogPathName = [System.Environment]::GetEnvironmentVariable($getEnvName) } diff --git a/README.md b/README.md index bcfe449..10e9e69 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,13 @@ -# ptrFiles - Protect, Transfer, Reconcile Files +# PeterDocs - Protect, Transfer, Reconcile Dcouments ## Summary -ptrFiles is for Protecting, Transfering and Reconciling Files on remote computer +PeterDocs is for Protecting, Transfering and Reconciling documents on remote computer where the computers are isolated or on different networks. The process uses a Windows PowerShell script and both the source and target computers that execute the code are required to be installed with Windows PowerShell. -The folder contents at source are archived and encrypted into a single file. You transfer the file to your target, where the content are unpacked using the decryption key. After archive contents are restored you can execute the reconcile function to veriy that the contents are transferred, unaltered. @@ -19,7 +18,7 @@ using tools such as: * Microsoft ROBOCOPY * rsync -Alternatively, you can use backup and restore utilities on the folder, and rely that +Alternatively, you can use backup and restore utilities on the folder, and rely that the contents are restored correctly. If you want this to be secure, ensure the backup is encrypted. @@ -32,8 +31,7 @@ JAM Software FileList. ## Background The script was born out a necessity to transfer a large volume of photographs -from one server to another, where shared network drives was not a feasible -solution. +from a media server to cloud backup. ## Usage @@ -41,13 +39,17 @@ Packages source folder contents into a 7ZIP file, adding a reconciliation file to the 7ZIP file and then encrypting the contents. Send * this script -* the 7ZIP package file -* plus optional SecretFilename ( if using RecipientKeyName ) to the target or recipient. +* the 7ZIP package file +* plus optional Secret File ( if using Recipient Key ) to the target or recipient. + +Alternatively you can direct the recipient to the PowerShell Gallery and ask them to +download the PeterDocs module and invoke the restore and reconcile commands from +within a PowerShell terminal window. The source folder is not altered and only read rights are required. A log -file is written at exceution to record activity. +file is written at execution to record activity. -The SecretFileName can be sent via email, while the 7ZIP can go different routes +The Secret File can be sent via email, while the 7ZIP can go different routes due to possible size such as: * Cloud storage provider @@ -68,9 +70,9 @@ and your documents have additonal protection. A log file is produced on execution. Repeated executions on the same day will add text content to the same log file. The default log name takes the form: -"ptr_files_yyyy-MM-dd.log" +"PETERDOCS_yyyy-MM-dd.log" -You will need to have installed the 7Zip4Powershell PowerShell cmdlet +You will need to have installed the 7Zip4Powershell PowerShell cmdlet before using the pack or unpack actions. You can install the cmdlet -by executing -.\ptrFiles.ps1 -Action install -Path ".\" +by executing +.\ptrDocs.ps1 -Action install -Path ".\" diff --git a/ReceiveArchive.md b/ReceiveArchive.md new file mode 100644 index 0000000..6c44844 --- /dev/null +++ b/ReceiveArchive.md @@ -0,0 +1,13 @@ +# Receive Archive + +## Why + +## When + +## How + +## What + +## Expand Usage + +Please read next the documentation on [expand the archive](Expand.md) diff --git a/Reconcile.md b/Reconcile.md new file mode 100644 index 0000000..01e5431 --- /dev/null +++ b/Reconcile.md @@ -0,0 +1,57 @@ +# Reconcile + +A reconcile file is generated as part of the Compress process and packed with the 7ZIP file. + +## Why + +When transferring or cloning documenmts to another location, you will want to +verify that the same documents have been restored unaltered at the destination. + +## When + +After the documents have been expanded and restored at the target +the next step to perform is reconcile the restored documents +against the reconcile file. + +## How + +The archive file and therefore the restore includes a reconciliation file +in the root folder. The reconciliation file is a CSV formatted file +listing all the documents and associated metadata. + +To perform the reconciliation you execute the ```Compare-Peter``` function. + +```powershell +Compare-Peter + -RestoreFolder + -ReconcileFile + -RootFolder + -ExtendedCheck + -LogPath +``` + +## What + +The reconciliation checks: + +1. Path to the document +2. Name of the document +3. Size of the document +4. Hash of document +5. Creation date and time of the document + +The document last update and time is not checked because the value +will reflect the date and time of restore. + +The reconciliation summary is displayed in the terminal and the log +wil lhave more information. + +If any errors are listed, please investigate the discrepancy. + +__Note__: For some restored documents the creation date and time may +have a variation of +/- 2 seconds and this is ignored by the reconciliation +process. + +## Finale + +Once you have reconciled the documents, you have completed the process. diff --git a/SendArchive.md b/SendArchive.md new file mode 100644 index 0000000..471c488 --- /dev/null +++ b/SendArchive.md @@ -0,0 +1,13 @@ +# Send Archive + +## Why + +## When + +## How + +## What + +## Receive Usage + +Please read next the documentation on [receiving the archive](ReceiveArchive.md) diff --git a/ptrFiles.ps1 b/ptrDocs.ps1 similarity index 53% rename from ptrFiles.ps1 rename to ptrDocs.ps1 index 246299c..12dfa0e 100644 --- a/ptrFiles.ps1 +++ b/ptrDocs.ps1 @@ -9,13 +9,13 @@ file to the 7ZIP file and then encrypting the contents. Send * this script * the 7ZIP package file - * plus optional SecretFilename ( if using RecipientKeyName ) + * plus optional SecretFile ( if using RecipientKey ) to the target or recipient. The source folder is not altered and only read rights are required. A log file is written at exceution to record activity. - The SecretFileName can be sent via email, while the 7ZIP can go different routes + The SecretFile can be sent via email, while the 7ZIP can go different routes due to possible size such as: * Cloud storage provider * HTTPS web file upload @@ -35,18 +35,17 @@ A log file is produced on execution. Repeated executions on the same day will add text content to the same log file. The default log name takes the form: - "ptr_files_yyyy-MM-dd.log" + "PETERDOCS_yyyy-MM-dd.log" You will need to have installed the 7Zip4Powershell PowerShell cmdlet before using the pack or unpack actions. You can install the cmdlet by executing - .\ptrFiles.ps1 -Action install -Path ".\" + .\ptrDocs.ps1 -Action install -Path ".\" - Author: Tom Peltonen .Parameter Action Action to perform, which can be: - - Install : Install 7Zip4PowerShell + - Install : Install 7Zip4PowerShell and other modules - Pack : Archive the contents of a folder(s) - Unpack : Unpack the archive, but no reconfile is performed - Reconcile : Reconcile the contents in the restored folder @@ -71,36 +70,34 @@ A file (@ prefix) containing a list of paths cannot contain generic path names, that is paths with trailing wildcard of "*" - .Parameter RecipientKeyName + .Parameter RecipientKey The recipient of the package which is used to find the appropriate - certificate for encrypting with the public key. Either the RecipientKeyName + certificate for encrypting with the public key. Either the RecipientKey or the SecretKey is required for packing or unpacking the 7ZIP file. - Using the RecipientKeyName is the most secure transfer option as a + Using the RecipientKey is the most secure transfer option as a asymmetric cryptographic key is used that can only be decrypted by the holder of the private key. - If you are using the RecipientKeyName, then the 7ZIP file contents can only - be unzipped by the holder of the private key and the SecretFileName file. + If you are using the RecipientKey, then the 7ZIP file contents can only + be unzipped by the holder of the private key and the SecretFile file. If you don't have the private, which you should not unless you are sending to yourself, then you cannot unpack the 7ZIP file. .Parameter SecretKey - A tradiitional secret to encrypt or decrypt the 7ZIP package. Either the RecipientKeyName + A tradiitional secret to encrypt or decrypt the 7ZIP package. Either the RecipientKey or the SecretKey is required for packing or unpacking the 7ZIP file. This method uses a symmetric cryptographic key exchange which is less secure then the - RecipientKeyName approach. + RecipientKey approach. - Note: Currently the script doe snot user Secure Strings + Note: Currently the script does not user Secure Strings - .Parameter ArchiveFileName + .Parameter ArchiveFile The location and name of the 7ZIP file. If not supplied a default 7ZIP file name will be generated in the current directory for the pack action. - The default name will take the form ".\transfer_protect_yyyyMMdd_hhmm.7z" - For unpack actions, the archive file name parameter is mandatory. - .Parameter RootFolderName + .Parameter RootFolder The root folder, which should be used if using wildcard (*) for the path. A guess will be made as to value if not supplied, which will work in many circumstances. @@ -110,8 +107,8 @@ An example to only include JPEG file is "*.jpg". You can also filter on picture file names starting with "IMG*.jpg" - .Parameter ReconcileFileName - The name of the reconfile file name to generate during pack or use + .Parameter ReconcileFile + The name of the reconcile file name to generate during pack or use during unpack. This is a file name without path. If no value is supplied, then a default name is generated. The reconcile file is included into the root of the 7ZIP file. @@ -120,21 +117,21 @@ The default name is "##protect_transfer_reconcile_files##.csv" - .Parameter SecretFileName - The secret file name is used with RecipientKeyName to secure the + .Parameter SecretFile + The secret file name is used with RecipientKey to secure the internally generated password for the 7ZIP file. When unpacking the - 7ZIP file you will need access to this file if RecipientKeyName + 7ZIP file you will need access to this file if RecipientKey was used. If not supplied a default name is used. This file is - encrypted with RecipientKeyName. + encrypted with RecipientKey. The default name is the archive file name with postfix ".key" .Parameter CloudProfile - The profile name to use for Install and Transfer actions. The - default for Install is "UserScope". The default for "Transfer" + The profile name to use for Install and Put/Get actions. The + default for Install is "UserScope". The default for "Put" or "GET" is "default" Profile name can also be specifed with Environment variable - "PTRFILES_PROFILE" + "PETERDOCS_PROFILE" .Parameter ExcludeHash Exclude the file hash from the reconcile file. As producing a file @@ -163,64 +160,58 @@ is not accessible and you want to reconcile, then this tool is appropriate. The following environment variables are supported: - - PTRFILES_RECIPIENTKEYNAME - - PTRFILES_PROFILE + - PETERDOCS_RECIPIENTKEY + - PETERDOCS_PROFILE + - PETERDOCS_ACCOUNTKEY + - PETERDOCS_LOGPATH .Example # Pack and encrypt all files in folder ".\transferpack\" using a private-public key # A file with the postifx ".key" is also generated alongside the 7ZIP file - .\ptrFiles.ps1 -Action pack -Path ".\transferpack\" -RecipientKeyName data@mycompany + .\ptrDocs.ps1 -Action pack -Path ".\transferpack\" -RecipientKey data@mycompany .Example # Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a private-public key # You will need the file "transfer_protect_yyyMMdd_hhmm.7z.key" to unpack the encrypted 7ZIP file - .\ptrFiles.ps1 -Action unpack -ArchiveFileName "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKeyName data@mycompany + .\ptrDocs.ps1 -Action unpack -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -RecipientKey data@mycompany .Example # Reconcile files in folder ".\targetdir" - .\ptrFiles.ps1 -Action reconcile -Path ".\targetdir" + .\ptrDocs.ps1 -Action reconcile -Path ".\targetdir" .Example # Pack and encrypt all files in folder ".\transferpack\" using a password - .\ptrFiles.ps1 -Action pack -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds" + .\ptrDocs.ps1 -Action pack -Path ".\transferpack\" -SecretKey "fjks932c-x=23ds" .Example # Unpack all files in 7ZIP file "transfer_protect_yyyMMdd_hhmm.7z" to folder ".\targetdir" using a password - .\ptrFiles.ps1 -Action unpack -ArchiveFileName "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds" + .\ptrDocs.ps1 -Action unpack -ArchiveFile "transfer_protect_yyyMMdd_hhmm.7z" -Path ".\targetdir" -SecretKey "fjks932c-x=23ds" .Example # Pack and encrypt all files in folder ".\transferpack\02*" where the folder name starts with "02" using a password - .\ptrFiles.ps1 -Action pack -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds" + .\ptrDocs.ps1 -Action pack -Path ".\transferpack\02*" -SecretKey "fjks932c-x=23ds" #> param ( [Parameter(Mandatory)][String] $Action, [Parameter(Mandatory)][String] $Path, - [String] $RecipientKeyName, + [String] $RecipientKey, [String] $SecretKey, - [String] $ArchiveFileName, - [String] $RootFolderName, + [String] $ArchiveFile, + [String] $RootFolder, [String] $FileFilter, - [String] $ReconcileFileName, - [String] $SecretFileName, + [String] $ReconcileFile, + [String] $SecretFile, [String] $CloudProfile, [switch] $ExcludeHash, [String] $LogPath ) -Import-Module .\PeterFiles +Import-Module .\PeterDocs -$default_dateLocal = Get-Date -Format "yyyyMMdd_HHmm" -$default_archiveFile = ".\ptr_file_##date##.7z" -$default_reconcileFile = "##protect_transfer_reconcile_files##.csv" - - - -# Main code logic starts here -function Invoke-Main { $actioned = $false @@ -230,212 +221,97 @@ function Invoke-Main { Install-Module -Name 7Zip4Powershell -Scope CurrentUser Install-Module -Name AWS.Tools.Installer -Scope CurrentUser Install-Module -Name AWS.Tools.S3 -Scope CurrentUser + Install-Module -Name Meerkat.PeterDocs -Scope CurrentUser } else { Install-Module -Name 7Zip4Powershell -Scope $cloudProfile Install-Module -Name AWS.Tools.Installer -Scope $cloudProfile Install-Module -Name AWS.Tools.S3 -Scope $cloudProfile + Install-Module -Name Meerkat.PeterDocs -Scope $cloudProfile } } if ($action -eq "Pack") { $actioned = $true - - if ($RecipientKeyName -eq "") { - $getEnvName = $(Get-SoftwareName) + "_RECIPIENTKEYNAME" - if ([System.Environment]::GetEnvironmentVariable($getEnvName) -ne "" -and $null -ne [System.Environment]::GetEnvironmentVariable($getEnvName)) { - $RecipientKeyName = [System.Environment]::GetEnvironmentVariable($getEnvName) - } - } - - if (($RecipientKeyName -eq "") -and ($SecretKey -eq "")) { - Write-Log "Recipient Key Name or Secret Key required for packing" - Write-Host "Recipient Key Name or Secret Key required for packing" -ForegroundColor Red - Close-Log - return - } - - if ($rootFolderName -eq "") { - if ($path.EndsWith("*")) { - Write-Log "Root folder required for packing when using wild card for Path" - Write-Host "Root folder required for packing when using wild card for Path" -ForegroundColor Red - Close-Log - return - } else { - $rootFolderName = $path - } - } - - if ($ArchiveFileName -eq "") { - $ArchiveFileName = $default_archiveFile.Replace("##date##", $default_dateLocal) - } - - if ($SecretKey -eq "") { - if ($secretFileName -eq "") - { - $secretFileName = $ArchiveFileName + ".key" - } - $secret = New-RandomPassword -Length 80 - Protect-CmsMessage -To $recipientKeyName -OutFile $secretFileName -Content $secret - } else { - $secret = $SecretKey - } - - Invoke-Pack -TransferFolder $path -Secret $secret -CompressFile $ArchiveFileName -ReconcileFile $reconcileFileName -RootFolder $rootFolderName -FileFilter $fileFilter + Compress-Peter -TransferFolder $path -Secret $secret -ArchiveFile $archiveFile -ReconcileFile $reconcileFile -RootFolder $rootFolder -FileFilter $fileFilter } if ($action -eq "Put") { - $actioned = $true - - if ($ArchiveFileName -eq "") { - Write-Log "Archive file name required" - Write-Host "Archive file name required" -ForegroundColor Red - Close-Log - return - } - - if (!(Test-Path -Path $ArchiveFileName )) { - Write-Log "Archive file '$ArchiveFileName' not found" - Write-Host "Archive file '$ArchiveFileName' not found" -ForegroundColor Red - Close-Log - return - } - - Invoke-PutArchive -CompressFile $archiveFileName -TargetPath $path -SecretFile $secretFileName -TargetProfile $cloudProfile + $actioned = $true + Send-Peter -ArchiveFile $archiveFile -TargetPath $path -SecretFile $secretFile -TargetProfile $cloudProfile } if ($action -eq "Get") { $actioned = $true - - if ($ArchiveFileName -eq "") { - Write-Log "Archive file name required" - Write-Host "Archive file name required" -ForegroundColor Red - Close-Log - return - } - - Invoke-GetArchive -CompressFile $archiveFileName -SourcePath $path -SecretFile $secretFileName -SourceProfile $cloudProfile + Receive-Peter -ArchiveFile $archiveFile -SourcePath $path -SecretFile $secretFile -SourceProfile $cloudProfile } if ($action -eq "Unpack") { $actioned = $true - - if ($RecipientKeyName -eq "") { - $getEnvName = $(Get-SoftwareName) + "_RECIPIENTKEYNAME" - if ([System.Environment]::GetEnvironmentVariable($getEnvName) -ne "" -and $null -ne [System.Environment]::GetEnvironmentVariable($getEnvName)) { - $RecipientKeyName = [System.Environment]::GetEnvironmentVariable($getEnvName) - } - } - - if (($RecipientKeyName -eq "") -and ($SecretKey -eq "")) { - Write-Log "Recipient Key Name or Secret Key required for unpacking" - Write-Host "Recipient Key Name or Secret Key required for unpacking" -ForegroundColor Red - Close-Log - return - } - if ($ArchiveFileName -eq "") { - Write-Log "Archive file Name required for unpacking" - Write-Host "Archive file Name required for unpacking" -ForegroundColor Red - Close-Log - return - } - - if ($SecretKey -eq "") { - if ($secretFileName -eq "") - { - $secretFileName = $ArchiveFileName + ".key" - } - $secret = Unprotect-CmsMessage -To $recipientKeyName -Path $secretFileName - } else { - $secret = $SecretKey - } - Invoke-Unpack -RestoreFolder $path -Secret $secret -CompressFile $ArchiveFileName - + Expand-Peter -RestoreFolder $path -Secret $secret -ArchiveFile $ArchiveFile } if ($action -eq "ReconcileFile") { $actioned = $true - if ($reconcileFileName -eq "") - { - $reconcileFileName = $default_reconcileFile - } - Set-Reconcile -ReconcileFile $reconcileFileName -FolderName $path -Feedback -RootFolderName $rootFolderName -FileFilter $fileFilter + Build-PeterReconcile -ReconcileFile $reconcileFile -FolderName $path -Feedback -RootFolder $rootFolder -FileFilter $fileFilter } if ($action -eq "Reconcile") { $actioned = $true - if ($reconcileFileName -eq "") - { - $reconcileFileName = $default_reconcileFile - } - $localReconcileFile = Join-Path -Path $path -ChildPath $reconcileFileName - Invoke-Reconcile -ReconcileFile $localReconcileFile -Folder $path -RootFolder $rootFolderName + Compare-Peter -ReconcileFile $reconcileFile -RestoreFolder $path -RootFolder $rootFolder } if ($action -eq "ArchiveInformation") { $actioned = $true - if (($RecipientKeyName -eq "") -and ($SecretKey -eq "")) { - Write-Log "Recipient Key Name or Secret Key required for 7Zip information" - Write-Host "Recipient Key Name or Secret Key required for 7Zip information" -ForegroundColor Red - Close-Log + if (($RecipientKey -eq "") -and ($SecretKey -eq "")) { + Write-Host "Recipient Key or Secret Key required for 7Zip information" -ForegroundColor Red return } if ($SecretKey -eq "") { - if ($secretFileName -eq "") + if ($SecretFile -eq "") { - $secretFileName = $ArchiveFileName + ".key" + $SecretFile = $ArchiveFileName + ".key" } - $secret = Unprotect-CmsMessage -To $recipientKeyName -Path $secretFileName + $secret = Unprotect-CmsMessage -To $RecipientKey -Path $SecretFile } else { $secret = $SecretKey } - Write-Log "Retrieving archive information" - Write-Host "Retrieving archive information" - - Get-7ZipInformation -ArchiveFileName $ArchiveFileName -Password $secret + Write-Host "Retrieving archive information" + Get-7ZipInformation -ArchiveFileName $ArchiveFile -Password $secret } if ($action -eq "MakeCert") { $actioned = $true - if (($RecipientKeyName -eq "") -and ($SecretKey -eq "")) { - Write-Log "Recipient Key Name required to create a standard certificate" + if (($RecipientKey -eq "") -and ($SecretKey -eq "")) { Write-Host "Recipient Key Name required to create a standard certificate" -ForegroundColor Red - Close-Log return } if ($Path -ne "Cert:\CurrentUser\My") { - Write-Log "The -Path value needs to be 'Cert:\CurrentUser\My'" Write-Host "The -Path value needs to be 'Cert:\CurrentUser\My'" -ForegroundColor Red - Close-Log return } - Write-Log "Making a file encryption certificate" - Write-Host "Making a file encryption certificate" - - New-SelfSignedCertificate -Subject $RecipientKeyName -KeyFriendlyName $RecipientKeyName -DnsName $RecipientKeyName -CertStoreLocation $Path -KeyUsage KeyEncipherment,DataEncipherment, KeyAgreement -Type DocumentEncryptionCert + Write-Host "Making a file encryption certificate" + New-SelfSignedCertificate -Subject $RecipientKey -KeyFriendlyName $RecipientKey -DnsName $RecipientKey -CertStoreLocation $Path -KeyUsage KeyEncipherment,DataEncipherment, KeyAgreement -Type DocumentEncryptionCert } if ($action -eq "ListCert") { $actioned = $true if ($Path -ne "Cert:\CurrentUser\My") { - Write-Log "The -Path value needs to be 'Cert:\CurrentUser\My'" Write-Host "The -Path value needs to be 'Cert:\CurrentUser\My'" -ForegroundColor Red - Close-Log return } - Write-Log "Listing encryption certificates" Write-Host "Listing encryption certificates" - if ($RecipientKeyName -eq "") + if ($RecipientKey -eq "") { Get-Childitem -Path $Path -DocumentEncryptionCert } else { @@ -445,7 +321,7 @@ function Invoke-Main { Write-Host "Thumbprint Subject" Write-Host "---------- -------" Get-Childitem -Path $Path -DocumentEncryptionCert | ForEach-Object { - if ($_.Subject -eq ("CN=$RecipientKeyName")) + if ($_.Subject -eq ("CN=$RecipientKey")) { Write-Host "$($_.Thumbprint) $($_.Subject)" } @@ -456,7 +332,6 @@ function Invoke-Main { if (!($actioned)) { - Write-Log "Unknown action '$action'. No processing performed" Write-Host "Unknown action '$action'. No processing performed" -ForegroundColor Red Write-Host "Recognised actions: " Write-Host " Pack : Pack folder contents into secure 7Zip file" @@ -470,29 +345,7 @@ function Invoke-Main { Write-Host "" Write-Host "For help use command " - Write-Host " Get-Help .\ptrFiles.ps1" + Write-Host " Get-Help .\ptrDocs.ps1" } - Close-Log -} - -$dateTimeStart = Get-Date -f "yyyy-MM-dd HH:mm:ss" -Write-Log "***********************************************************************************" -Write-Log "* Start of processing: [$dateTimeStart]" -Write-Log "***********************************************************************************" - - -Write-Log "Script parameters follow" -ForEach ($boundParam in $PSBoundParameters.GetEnumerator()) -{ - if ($boundParam.Key -eq "SecretKey") { - Write-Log "Parameter: $($boundParam.Key) Value: ************** " - } else { - Write-Log "Parameter: $($boundParam.Key) Value: $($boundParam.Value) " - } -} -Write-Log "" - - -Invoke-Main