From b6fa34c9c6c13df35a3638c866e333b41d031f1f Mon Sep 17 00:00:00 2001 From: meerkat Date: Sun, 10 Oct 2021 23:09:57 +1100 Subject: [PATCH] BSB Sample code --- docs/samples/powershell/Invoke-BSBSample.ps1 | 76 ++++++-- source/powershell/Get-Marti.ps1 | 49 +++-- source/powershell/Get-MartiFileAttributes.ps1 | 44 +++-- source/powershell/New-Marti.ps1 | 180 ------------------ source/powershell/New-MartiChildItem.ps1 | 58 ++++++ .../{Add-MartiItem.ps1 => New-MartiItem.ps1} | 4 +- 6 files changed, 192 insertions(+), 219 deletions(-) create mode 100644 source/powershell/New-MartiChildItem.ps1 rename source/powershell/{Add-MartiItem.ps1 => New-MartiItem.ps1} (96%) diff --git a/docs/samples/powershell/Invoke-BSBSample.ps1 b/docs/samples/powershell/Invoke-BSBSample.ps1 index 49a1b36..4f543a2 100644 --- a/docs/samples/powershell/Invoke-BSBSample.ps1 +++ b/docs/samples/powershell/Invoke-BSBSample.ps1 @@ -1,6 +1,7 @@ . C:\Users\meerkat\source\marti\source\powershell\New-Marti.ps1 -. C:\Users\meerkat\source\marti\source\powershell\Add-MartiItem.ps1 +. C:\Users\meerkat\source\marti\source\powershell\New-MartiChildItem.ps1 +. C:\Users\meerkat\source\marti\source\powershell\New-MartiItem.ps1 . C:\Users\meerkat\source\marti\source\powershell\Get-Marti.ps1 . C:\Users\meerkat\source\marti\source\powershell\Compress-Marti.ps1 . C:\Users\meerkat\source\marti\source\powershell\Get-MartiFileAttributes.ps1 @@ -85,13 +86,30 @@ Write-Host "First fetch the BSB files " -ForeGroundColor Green $fileList = ListFtpDirectory -Username "anonymous" -Password "anon@merebox.com" -RemoteFile $remoteDirectory Write-Host "File list size: $($fileList.count)" + +Write-Host "Now iterate through the remote files and build remote marti list " -ForeGroundColor Green + $oMarti = New-MartiDefinition +$oMarti.title = "Remote_BSB_data" +$oMarti.description = "This definition covers the remote BSB data files `r downloaded from the Australian Payment Network" +$oMarti.contactPoint = "meerkat@merebox.com" +$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json" +$oMarti.theme = "payment" + ForEach ($item in $fileList) { if ($item -ne "" -and $item.startswith("BSBDirectory")) { - #Write-Host "Pulling file: $item" PullFtpFile -Username "anonymous" -Password "anon@merebox.com" -RemoteFile ($remoteDirectory + $item) -OutputPath (Join-Path -Path $localDirectory -ChildPath $item) Write-Host "Add BSB $item file to Remote marti metadata sample " -ForeGroundColor Yellow - $oResource = Add-MartiItem -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes + $oResource = New-MartiItem -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes + if ($item.endswith(".txt")) { + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8 + } + if ($item.endswith(".csv")) { + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0 + } $oMarti.resources += $oResource } } @@ -100,21 +118,51 @@ $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBRemote.mri.json" $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson Write-Host "Remote marti definition file is $fileJson " -ForeGroundColor Green -Write-Host "Now iterate through the local files and build ZIP " -ForeGroundColor Green -if ($fileList -lt 0) { - $zipFile = Join-Path -Path $localDirectory -ChildPath "BSBDirectory.zip" - if (Test-Path -Path $zipFile) { - Remove-Item -Path $zipFile - } - foreach($file in Get-ChildItem $localDirectory) - { - if ($file.Name.startswith("BSBDirectory")) { - Write-Host "Add BSB file $file to Local marti metadata sample " -ForeGroundColor Yellow - Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update +Write-Host "Now iterate through the local files and build marti ZIP " -ForeGroundColor Green + +$oMarti = New-MartiDefinition +$oMarti.title = "Zip_BSB_data" +$oMarti.description = "This definition covers the ZIP BSB data files `r downloaded from the Australian Payment Network" +$oMarti.contactPoint = "meerkat@merebox.com" +$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json" +$oMarti.theme = "payment" + +$zipFileName = "BSBDirectory.zip" +$zipFile = Join-Path -Path $localDirectory -ChildPath $zipFileName +if (Test-Path -Path $zipFile) { + Remove-Item -Path $zipFile +} +foreach($file in Get-ChildItem $localDirectory) +{ + if ($file.Name.startswith("BSBDirectory") -and $file.Name -ne $zipFileName) { + Write-Host "Add BSB file $file to Local marti metadata sample " -ForeGroundColor Yellow + Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update + $oResource = New-MartiItem -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes + if ($file.Extension -eq ".txt") { + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8 } + if ($file.Extension -eq ".csv") { + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0 + } + $oResource.url = "@"+$zipFileName + "/" + $file.Name + $oMarti.resources += $oResource } } +$oResource = New-MartiItem -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes +Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "WINZIP" +$oMarti.resources += $oResource + +$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mri.json" +$oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson +Write-Host "ZIP marti definition file is $fileJson " -ForeGroundColor Green + + + +Write-Host "Now iterate through the local files with ZIP " -ForeGroundColor Green $oMarti = New-MartiChildItem -SourceFolder $localDirectory -UrlPath "./test" -Filter "BSBDirectory*" -LogPath ".\test\Logs" -ExtendAttributes $oMarti.title = "Local_BSB_data" diff --git a/source/powershell/Get-Marti.ps1 b/source/powershell/Get-Marti.ps1 index 5ee4020..4534209 100644 --- a/source/powershell/Get-Marti.ps1 +++ b/source/powershell/Get-Marti.ps1 @@ -64,13 +64,13 @@ function Get-SoftwareName { -function Get-MartiItem +function Get-MartChildResource { Param( - [Parameter(Mandatory)][PSCustomObject] $MartiDefintiion, + [Parameter(Mandatory)][PSCustomObject] $Marti, [Parameter(Mandatory)][String] $Title, - [String] $DocumentName, - [String] $Format, + [String] $ResourceName, + [String] $Format = "*", [String] $LogPath ) @@ -80,12 +80,12 @@ function Get-MartiItem Write-Debug "Parameter: LogPath Value: $LogPath " Open-Log Write-Log "Function 'Get-MartiItem' parameters follow" - Write-Log "Parameter: DocumentName Value: $DocumentName " - Write-Log "Parameter: Filter Value: $Filter " + Write-Log "Parameter: ResourceName Value: $ResourceName " + Write-Log "Parameter: Format Value: $Format " Write-Log "" - if ($null -eq $MartiDefintiion) { + if ($null -eq $Marti) { $Global:MartiErrorId = "MRI2101" $message = "No Marti definition supplied" Write-Log ($message + " " + $Global:MartiErrorId) @@ -94,7 +94,7 @@ function Get-MartiItem } - if ($null -eq $MartiDefintiion.resources -or $MartiDefintiion.resources.Count -lt 1) { + if ($null -eq $Marti.resources -or $Marti.resources.Count -lt 1) { $Global:MartiErrorId = "MRI2102" $message = "No documents listed" Write-Log ($message + " " + $Global:MartiErrorId) @@ -104,13 +104,13 @@ function Get-MartiItem [System.Collections.ArrayList]$lresource = @() - $MartiDefintiion.resources | ForEach-Object { + $Marti.resources | ForEach-Object { - if ($null -eq $Format -or $Format -eq "*" -or $Format -eq $_.format ) { + if ($Format -eq "*" -or $Format -eq $_.format ) { if ($Title -ne "*" -and $_.title -eq $Title) { $lresource += $_ } else { - if ($DocumentName -ne "*" -and $_.documentName -eq $DocumentName) { + if ($ResourceName -ne "*" -and $_.documentName -eq $ResourceName ) { $lresource += $_ } } @@ -122,3 +122,30 @@ function Get-MartiItem return $lresource } + +function Get-MartiResource { + Param ( + # Marti definition + [Parameter(Mandatory)] [PSCustomObject] $Marti, + # Resource ID + [Parameter(Mandatory)] [String] $ResourceName + ) + + $script:LogPathName = $LogPath + + Write-Debug "Parameter: LogPath Value: $LogPath " + Open-Log + Write-Log "Function 'Get-MartiResource' parameters follow" + Write-Log "Parameter: ResourceName Value: $ResourceName " + Write-Log "" + + foreach ($item in $oMarti.resources) { + if ($item.uid -eq $ResourceName -or $item.documentName -eq $ResourceName){ + Close-Log + return $item + } + } + + Close-Log + return $null +} diff --git a/source/powershell/Get-MartiFileAttributes.ps1 b/source/powershell/Get-MartiFileAttributes.ps1 index a694c1f..6ae619f 100644 --- a/source/powershell/Get-MartiFileAttributes.ps1 +++ b/source/powershell/Get-MartiFileAttributes.ps1 @@ -145,10 +145,10 @@ function New-DefaultZipAttributes { return $lattribute } -function Update-AttributeValueString { +function Set-AttributeValueString { Param ( # Attribute List - [Parameter(Mandatory)] [System.Collections.ArrayList] $lAttribute, + [Parameter(Mandatory)] [System.Collections.ArrayList] $Attributes, # Attribute Category [Parameter(Mandatory)] [String] $Category, # Attribute Key Name @@ -161,7 +161,7 @@ function Update-AttributeValueString { [String] $Comparison = "EQ" ) - foreach ($item in $lAttribute) + foreach ($item in $Attributes) { if ($item.category -eq $Category -and $item.name -eq $Key -and $item.function -eq $Function) { @@ -173,11 +173,20 @@ function Update-AttributeValueString { } } - Write-Error "No match found for attribute name '$Key' and category '$Category' and function '$Function' " + # Add the attribute + $oAttribute = [PSCustomObject]@{ + category = $Category + name = $Key + function = $Function + comparison = $Comparison + value = $Value + } + $Attributes += $oAttribute + return } -function Update-AttributeValueNumber { +function Set-AttributeValueNumber { Param ( # Attribute List [Parameter(Mandatory)] [System.Collections.ArrayList] $Attributes, @@ -205,11 +214,21 @@ function Update-AttributeValueNumber { } } - Write-Error "No match found for attribute name '$Key' and category '$Category' and function '$Function' " + + # Add the attribute + $oAttribute = [PSCustomObject]@{ + category = $Category + name = $Key + function = $Function + comparison = $Comparison + value = $Value + } + $Attributes += $oAttribute + return } -function Get-MartiFileAttributes { +function Set-MartiFileAttributes { Param ( # File path [Parameter(Mandatory)] [String] $Path, @@ -235,8 +254,8 @@ function Get-MartiFileAttributes { } $rowCount += 1 } - Update-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount - Update-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount + Set-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount + Set-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount } } @@ -256,8 +275,8 @@ function Get-MartiFileAttributes { } $rowCount += 1 } - Update-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount - Update-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount + Set-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount + Set-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount } } @@ -271,7 +290,7 @@ function Get-MartiFileAttributes { $shell = New-Object -Com Shell.Application $zipFile = $shell.NameSpace($Path) $items = $zipFile.Items() - Update-AttributeValueNumber -Attributes $lattribute -Key "files" -Category "dataset" -Function "count" -Value $items.Count + Set-AttributeValueNumber -Attributes $lattribute -Key "files" -Category "dataset" -Function "count" -Value $items.Count } } @@ -286,3 +305,4 @@ function Get-MartiFileAttributes { return $lattribute } + diff --git a/source/powershell/New-Marti.ps1 b/source/powershell/New-Marti.ps1 index f94ee8c..37df515 100644 --- a/source/powershell/New-Marti.ps1 +++ b/source/powershell/New-Marti.ps1 @@ -106,94 +106,6 @@ function New-MartiDefinition } - - -function New-MartiChildItem -{ -Param( - [Parameter(Mandatory)][String] $SourceFolder, - [String] $Filter ="*", - [String] $UrlPath, - [switch] $Recurse, - [switch] $ExtendAttributes, - [switch] $ExcludeHash, - [String] $LogPath - -) - $script:LogPathName = $LogPath - - Write-Debug "Parameter: LogPath Value: $LogPath " - Open-Log - Write-Log "Function 'New-MartiDefinition' parameters follow" - Write-Log "Parameter: SourceFolder Value: $SourceFolder " - Write-Log "Parameter: Filter Value: $Filter " - Write-Log "Parameter: Recurse Value: $Recurse " - Write-Log "Parameter: ExcludeHash Value: $ExcludeHash " - Write-Log "" - - if ($ExcludeHash) { - $hashAlgo = "" - } - else { - $hashAlgo = "SHA256" - } - $version = "1.1.0" - - $oMarti = New-MartiDefinition - $lresource = $oMarti.resources - - $SourceFullName = (Get-Item -Path $SourceFolder).FullName - - Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object { - - Write-Log "Define file $($_.FullName) " - if ($ExcludeHash) { - $hash = "" - } else { - $hash = (Get-FileHash -Path $_.FullName -Algorithm $hashAlgo).Hash - } - - $lattribute = Get-MartiFileAttributes -Path $_.FullName -FileType $_.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes - - $oResource = [PSCustomObject]@{ - title = $_.Name.Replace($_.Extension, "") - uid = (New-Guid).ToString() - documentName = $_.Name - issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss" - modified = $_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss") - state = "active" - author = "" - length = $_.Length - hash = $hash - hashAlgo = $hashAlgo - - description = "" - url = "" - version = $version - format = $_.Extension.Substring(1) - compression = "" - encryption = "" - - attributes = $lattribute - } - - if ($null -ne $UrlPath -and $UrlPath -ne "") { - $postfixName = $_.FullName.Replace($SourceFullName, "") - $oResource.url = Join-Path -Path $UrlPath -ChildPath $postfixName - } - - $lresource += $oResource - - } - Write-Log "Captured $($lresource.Count) items" - $oMarti.resources = $lresource - - Close-Log - - return $oMarti - -} - function Set-MartiAttribute { Param( @@ -233,95 +145,3 @@ Param( return $Attributes } -function New-MartiItem -{ -Param( - [Parameter(Mandatory)][String] $SourcePath, - [String] $UrlPath = "", - [switch] $ExcludeHash, - [switch] $ExtendAttributes, - [String] $LogPath - -) - $Global:MartiErrorId = "" - $script:LogPathName = $LogPath - - Write-Debug "Parameter: LogPath Value: $LogPath " - Open-Log - Write-Log "Function 'New-MartiItem' parameters follow" - Write-Log "Parameter: SourcePath Value: $SourcePath " - Write-Log "Parameter: ExcludeHash Value: $ExcludeHash " - Write-Log "" - - - if ($ExcludeHash) { - $hashAlgo = "" - } - else { - $hashAlgo = "SHA256" - } - $version = "1.1.0" - - $oMarti = New-MartiDefinition - $lresource = $oMarti.resources - - if (Test-Path -Path $SourcePath -PathType Leaf) { - - $item = Get-Item -Path $SourcePath -Force - - Write-Log "Define file $($item.FullName) " - if ($ExcludeHash) { - $hash = "" - } else { - $hash = (Get-FileHash -Path $item.FullName -Algorithm $hashAlgo).Hash - } - - $lattribute = Get-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes - - $oResource = [PSCustomObject]@{ - title = $item.Name.Replace($item.Extension, "") - uid = (New-Guid).ToString() - documentName = $item.Name - issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss" - modified = $item.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss") - state = "active" - author = "" - length = $item.Length - hash = $hash - hashAlgo = $hashAlgo - - description = "" - url = "" - version = $version - format = $item.Extension.Substring(1) - compression = "" - encryption = "" - - attributes = $lattribute - } - - if ($null -ne $UrlPath -and $UrlPath -ne "") { - if ($UrlPath[$UrlPath.Length-1] -eq "/" -or $UrlPath[$UrlPath.Length-1] -eq "\\") { - $oResource.url = $UrlPath.Replace("\\", "/") + "/" + $_.Name - } else { - $oResource.url = $UrlPath.Replace("\\", "/") + "/" + $_.Name - } - } - - $lresource += $oResource - - } else { - $Global:MartiErrorId = "MRI2001" - $message = "Document '$SourcePath' not found or is a folder" - Write-Log ($message + " " + $Global:MartiErrorId) - Close-Log - throw $message - } - Write-Log "Captured $($lresource.Count) items" - $oMarti.resources = $lresource - Close-Log - - return $oMarti - -} - diff --git a/source/powershell/New-MartiChildItem.ps1 b/source/powershell/New-MartiChildItem.ps1 new file mode 100644 index 0000000..4ed4a95 --- /dev/null +++ b/source/powershell/New-MartiChildItem.ps1 @@ -0,0 +1,58 @@ + +function New-MartiChildItem +{ +Param( + [Parameter(Mandatory)][String] $SourceFolder, + [String] $Filter ="*", + [String] $UrlPath, + [switch] $Recurse, + [switch] $ExtendAttributes, + [switch] $ExcludeHash, + [String] $LogPath + +) + $script:LogPathName = $LogPath + + Write-Debug "Parameter: LogPath Value: $LogPath " + Open-Log + Write-Log "Function 'New-MartiDefinition' parameters follow" + Write-Log "Parameter: SourceFolder Value: $SourceFolder " + Write-Log "Parameter: Filter Value: $Filter " + Write-Log "Parameter: Recurse Value: $Recurse " + Write-Log "Parameter: ExtendAttributes Value: $ExtendAttributes " + Write-Log "Parameter: ExcludeHash Value: $ExcludeHash " + Write-Log "" + + $oMarti = New-MartiDefinition + $lresource = $oMarti.resources + + $SourceFullName = (Get-Item -Path $SourceFolder).FullName + + Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object { + + $oResource = New-MartiItem -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash + + if ($null -ne $UrlPath -and $UrlPath -ne "") { + $postfixName = $_.FullName.Replace($SourceFullName, "") + if ($postfixName[0] -eq "/" -or $postfixName[0] -eq "`\" ){ + $postfixName = $postfixName.Substring(1, ($postfixName.Length-1)) + } + if ($UrlPath[$UrlPath.Length-1] -eq "/" -or $UrlPath[$UrlPath.Length-1] -eq "`\") { + $oResource.url = $UrlPath.Replace("`\", "/") + $postfixName.Replace("`\", "/") + } else { + $oResource.url = $UrlPath.Replace("`\", "/") + "/" + $postfixName.Replace("`\", "/") + } + } + + $lresource += $oResource + + } + Write-Log "Captured $($lresource.Count) items" + $oMarti.resources = $lresource + + Close-Log + + return $oMarti + +} + diff --git a/source/powershell/Add-MartiItem.ps1 b/source/powershell/New-MartiItem.ps1 similarity index 96% rename from source/powershell/Add-MartiItem.ps1 rename to source/powershell/New-MartiItem.ps1 index 59273f2..91d4ed4 100644 --- a/source/powershell/Add-MartiItem.ps1 +++ b/source/powershell/New-MartiItem.ps1 @@ -1,5 +1,5 @@ -function Add-MartiItem +function New-MartiItem { Param( [Parameter(Mandatory)][String] $SourcePath, @@ -38,7 +38,7 @@ Param( $hash = (Get-FileHash -Path $item.FullName -Algorithm $hashAlgo).Hash } - $lattribute = Get-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes + $lattribute = Set-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes $oResource = [PSCustomObject]@{ title = $item.Name.Replace($item.Extension, "")