BSB Sample code

draft_specifications
meerkat 2021-10-10 23:09:57 +11:00
parent 39fa406872
commit b6fa34c9c6
6 changed files with 192 additions and 219 deletions

View File

@ -1,6 +1,7 @@
. C:\Users\meerkat\source\marti\source\powershell\New-Marti.ps1
. C:\Users\meerkat\source\marti\source\powershell\Add-MartiItem.ps1
. C:\Users\meerkat\source\marti\source\powershell\New-MartiChildItem.ps1
. C:\Users\meerkat\source\marti\source\powershell\New-MartiItem.ps1
. C:\Users\meerkat\source\marti\source\powershell\Get-Marti.ps1
. C:\Users\meerkat\source\marti\source\powershell\Compress-Marti.ps1
. C:\Users\meerkat\source\marti\source\powershell\Get-MartiFileAttributes.ps1
@ -85,13 +86,30 @@ Write-Host "First fetch the BSB files " -ForeGroundColor Green
$fileList = ListFtpDirectory -Username "anonymous" -Password "anon@merebox.com" -RemoteFile $remoteDirectory
Write-Host "File list size: $($fileList.count)"
Write-Host "Now iterate through the remote files and build remote marti list " -ForeGroundColor Green
$oMarti = New-MartiDefinition
$oMarti.title = "Remote_BSB_data"
$oMarti.description = "This definition covers the remote BSB data files `r downloaded from the Australian Payment Network"
$oMarti.contactPoint = "meerkat@merebox.com"
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json"
$oMarti.theme = "payment"
ForEach ($item in $fileList) {
if ($item -ne "" -and $item.startswith("BSBDirectory")) {
#Write-Host "Pulling file: $item"
PullFtpFile -Username "anonymous" -Password "anon@merebox.com" -RemoteFile ($remoteDirectory + $item) -OutputPath (Join-Path -Path $localDirectory -ChildPath $item)
Write-Host "Add BSB $item file to Remote marti metadata sample " -ForeGroundColor Yellow
$oResource = Add-MartiItem -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes
$oResource = New-MartiItem -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes
if ($item.endswith(".txt")) {
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8
}
if ($item.endswith(".csv")) {
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0
}
$oMarti.resources += $oResource
}
}
@ -100,21 +118,51 @@ $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBRemote.mri.json"
$oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
Write-Host "Remote marti definition file is $fileJson " -ForeGroundColor Green
Write-Host "Now iterate through the local files and build ZIP " -ForeGroundColor Green
if ($fileList -lt 0) {
$zipFile = Join-Path -Path $localDirectory -ChildPath "BSBDirectory.zip"
if (Test-Path -Path $zipFile) {
Remove-Item -Path $zipFile
}
foreach($file in Get-ChildItem $localDirectory)
{
if ($file.Name.startswith("BSBDirectory")) {
Write-Host "Add BSB file $file to Local marti metadata sample " -ForeGroundColor Yellow
Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update
Write-Host "Now iterate through the local files and build marti ZIP " -ForeGroundColor Green
$oMarti = New-MartiDefinition
$oMarti.title = "Zip_BSB_data"
$oMarti.description = "This definition covers the ZIP BSB data files `r downloaded from the Australian Payment Network"
$oMarti.contactPoint = "meerkat@merebox.com"
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json"
$oMarti.theme = "payment"
$zipFileName = "BSBDirectory.zip"
$zipFile = Join-Path -Path $localDirectory -ChildPath $zipFileName
if (Test-Path -Path $zipFile) {
Remove-Item -Path $zipFile
}
foreach($file in Get-ChildItem $localDirectory)
{
if ($file.Name.startswith("BSBDirectory") -and $file.Name -ne $zipFileName) {
Write-Host "Add BSB file $file to Local marti metadata sample " -ForeGroundColor Yellow
Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update
$oResource = New-MartiItem -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
if ($file.Extension -eq ".txt") {
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8
}
if ($file.Extension -eq ".csv") {
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0
}
$oResource.url = "@"+$zipFileName + "/" + $file.Name
$oMarti.resources += $oResource
}
}
$oResource = New-MartiItem -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "WINZIP"
$oMarti.resources += $oResource
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mri.json"
$oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
Write-Host "ZIP marti definition file is $fileJson " -ForeGroundColor Green
Write-Host "Now iterate through the local files with ZIP " -ForeGroundColor Green
$oMarti = New-MartiChildItem -SourceFolder $localDirectory -UrlPath "./test" -Filter "BSBDirectory*" -LogPath ".\test\Logs" -ExtendAttributes
$oMarti.title = "Local_BSB_data"

View File

@ -64,13 +64,13 @@ function Get-SoftwareName {
function Get-MartiItem
function Get-MartChildResource
{
Param(
[Parameter(Mandatory)][PSCustomObject] $MartiDefintiion,
[Parameter(Mandatory)][PSCustomObject] $Marti,
[Parameter(Mandatory)][String] $Title,
[String] $DocumentName,
[String] $Format,
[String] $ResourceName,
[String] $Format = "*",
[String] $LogPath
)
@ -80,12 +80,12 @@ function Get-MartiItem
Write-Debug "Parameter: LogPath Value: $LogPath "
Open-Log
Write-Log "Function 'Get-MartiItem' parameters follow"
Write-Log "Parameter: DocumentName Value: $DocumentName "
Write-Log "Parameter: Filter Value: $Filter "
Write-Log "Parameter: ResourceName Value: $ResourceName "
Write-Log "Parameter: Format Value: $Format "
Write-Log ""
if ($null -eq $MartiDefintiion) {
if ($null -eq $Marti) {
$Global:MartiErrorId = "MRI2101"
$message = "No Marti definition supplied"
Write-Log ($message + " " + $Global:MartiErrorId)
@ -94,7 +94,7 @@ function Get-MartiItem
}
if ($null -eq $MartiDefintiion.resources -or $MartiDefintiion.resources.Count -lt 1) {
if ($null -eq $Marti.resources -or $Marti.resources.Count -lt 1) {
$Global:MartiErrorId = "MRI2102"
$message = "No documents listed"
Write-Log ($message + " " + $Global:MartiErrorId)
@ -104,13 +104,13 @@ function Get-MartiItem
[System.Collections.ArrayList]$lresource = @()
$MartiDefintiion.resources | ForEach-Object {
$Marti.resources | ForEach-Object {
if ($null -eq $Format -or $Format -eq "*" -or $Format -eq $_.format ) {
if ($Format -eq "*" -or $Format -eq $_.format ) {
if ($Title -ne "*" -and $_.title -eq $Title) {
$lresource += $_
} else {
if ($DocumentName -ne "*" -and $_.documentName -eq $DocumentName) {
if ($ResourceName -ne "*" -and $_.documentName -eq $ResourceName ) {
$lresource += $_
}
}
@ -122,3 +122,30 @@ function Get-MartiItem
return $lresource
}
function Get-MartiResource {
Param (
# Marti definition
[Parameter(Mandatory)] [PSCustomObject] $Marti,
# Resource ID
[Parameter(Mandatory)] [String] $ResourceName
)
$script:LogPathName = $LogPath
Write-Debug "Parameter: LogPath Value: $LogPath "
Open-Log
Write-Log "Function 'Get-MartiResource' parameters follow"
Write-Log "Parameter: ResourceName Value: $ResourceName "
Write-Log ""
foreach ($item in $oMarti.resources) {
if ($item.uid -eq $ResourceName -or $item.documentName -eq $ResourceName){
Close-Log
return $item
}
}
Close-Log
return $null
}

View File

@ -145,10 +145,10 @@ function New-DefaultZipAttributes {
return $lattribute
}
function Update-AttributeValueString {
function Set-AttributeValueString {
Param (
# Attribute List
[Parameter(Mandatory)] [System.Collections.ArrayList] $lAttribute,
[Parameter(Mandatory)] [System.Collections.ArrayList] $Attributes,
# Attribute Category
[Parameter(Mandatory)] [String] $Category,
# Attribute Key Name
@ -161,7 +161,7 @@ function Update-AttributeValueString {
[String] $Comparison = "EQ"
)
foreach ($item in $lAttribute)
foreach ($item in $Attributes)
{
if ($item.category -eq $Category -and $item.name -eq $Key -and $item.function -eq $Function)
{
@ -173,11 +173,20 @@ function Update-AttributeValueString {
}
}
Write-Error "No match found for attribute name '$Key' and category '$Category' and function '$Function' "
# Add the attribute
$oAttribute = [PSCustomObject]@{
category = $Category
name = $Key
function = $Function
comparison = $Comparison
value = $Value
}
$Attributes += $oAttribute
return
}
function Update-AttributeValueNumber {
function Set-AttributeValueNumber {
Param (
# Attribute List
[Parameter(Mandatory)] [System.Collections.ArrayList] $Attributes,
@ -205,11 +214,21 @@ function Update-AttributeValueNumber {
}
}
Write-Error "No match found for attribute name '$Key' and category '$Category' and function '$Function' "
# Add the attribute
$oAttribute = [PSCustomObject]@{
category = $Category
name = $Key
function = $Function
comparison = $Comparison
value = $Value
}
$Attributes += $oAttribute
return
}
function Get-MartiFileAttributes {
function Set-MartiFileAttributes {
Param (
# File path
[Parameter(Mandatory)] [String] $Path,
@ -235,8 +254,8 @@ function Get-MartiFileAttributes {
}
$rowCount += 1
}
Update-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount
Update-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount
Set-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount
Set-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount
}
}
@ -256,8 +275,8 @@ function Get-MartiFileAttributes {
}
$rowCount += 1
}
Update-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount
Update-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount
Set-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount
Set-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount
}
}
@ -271,7 +290,7 @@ function Get-MartiFileAttributes {
$shell = New-Object -Com Shell.Application
$zipFile = $shell.NameSpace($Path)
$items = $zipFile.Items()
Update-AttributeValueNumber -Attributes $lattribute -Key "files" -Category "dataset" -Function "count" -Value $items.Count
Set-AttributeValueNumber -Attributes $lattribute -Key "files" -Category "dataset" -Function "count" -Value $items.Count
}
}
@ -286,3 +305,4 @@ function Get-MartiFileAttributes {
return $lattribute
}

View File

@ -106,94 +106,6 @@ function New-MartiDefinition
}
function New-MartiChildItem
{
Param(
[Parameter(Mandatory)][String] $SourceFolder,
[String] $Filter ="*",
[String] $UrlPath,
[switch] $Recurse,
[switch] $ExtendAttributes,
[switch] $ExcludeHash,
[String] $LogPath
)
$script:LogPathName = $LogPath
Write-Debug "Parameter: LogPath Value: $LogPath "
Open-Log
Write-Log "Function 'New-MartiDefinition' parameters follow"
Write-Log "Parameter: SourceFolder Value: $SourceFolder "
Write-Log "Parameter: Filter Value: $Filter "
Write-Log "Parameter: Recurse Value: $Recurse "
Write-Log "Parameter: ExcludeHash Value: $ExcludeHash "
Write-Log ""
if ($ExcludeHash) {
$hashAlgo = ""
}
else {
$hashAlgo = "SHA256"
}
$version = "1.1.0"
$oMarti = New-MartiDefinition
$lresource = $oMarti.resources
$SourceFullName = (Get-Item -Path $SourceFolder).FullName
Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object {
Write-Log "Define file $($_.FullName) "
if ($ExcludeHash) {
$hash = ""
} else {
$hash = (Get-FileHash -Path $_.FullName -Algorithm $hashAlgo).Hash
}
$lattribute = Get-MartiFileAttributes -Path $_.FullName -FileType $_.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
$oResource = [PSCustomObject]@{
title = $_.Name.Replace($_.Extension, "")
uid = (New-Guid).ToString()
documentName = $_.Name
issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
modified = $_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")
state = "active"
author = ""
length = $_.Length
hash = $hash
hashAlgo = $hashAlgo
description = ""
url = ""
version = $version
format = $_.Extension.Substring(1)
compression = ""
encryption = ""
attributes = $lattribute
}
if ($null -ne $UrlPath -and $UrlPath -ne "") {
$postfixName = $_.FullName.Replace($SourceFullName, "")
$oResource.url = Join-Path -Path $UrlPath -ChildPath $postfixName
}
$lresource += $oResource
}
Write-Log "Captured $($lresource.Count) items"
$oMarti.resources = $lresource
Close-Log
return $oMarti
}
function Set-MartiAttribute
{
Param(
@ -233,95 +145,3 @@ Param(
return $Attributes
}
function New-MartiItem
{
Param(
[Parameter(Mandatory)][String] $SourcePath,
[String] $UrlPath = "",
[switch] $ExcludeHash,
[switch] $ExtendAttributes,
[String] $LogPath
)
$Global:MartiErrorId = ""
$script:LogPathName = $LogPath
Write-Debug "Parameter: LogPath Value: $LogPath "
Open-Log
Write-Log "Function 'New-MartiItem' parameters follow"
Write-Log "Parameter: SourcePath Value: $SourcePath "
Write-Log "Parameter: ExcludeHash Value: $ExcludeHash "
Write-Log ""
if ($ExcludeHash) {
$hashAlgo = ""
}
else {
$hashAlgo = "SHA256"
}
$version = "1.1.0"
$oMarti = New-MartiDefinition
$lresource = $oMarti.resources
if (Test-Path -Path $SourcePath -PathType Leaf) {
$item = Get-Item -Path $SourcePath -Force
Write-Log "Define file $($item.FullName) "
if ($ExcludeHash) {
$hash = ""
} else {
$hash = (Get-FileHash -Path $item.FullName -Algorithm $hashAlgo).Hash
}
$lattribute = Get-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
$oResource = [PSCustomObject]@{
title = $item.Name.Replace($item.Extension, "")
uid = (New-Guid).ToString()
documentName = $item.Name
issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
modified = $item.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")
state = "active"
author = ""
length = $item.Length
hash = $hash
hashAlgo = $hashAlgo
description = ""
url = ""
version = $version
format = $item.Extension.Substring(1)
compression = ""
encryption = ""
attributes = $lattribute
}
if ($null -ne $UrlPath -and $UrlPath -ne "") {
if ($UrlPath[$UrlPath.Length-1] -eq "/" -or $UrlPath[$UrlPath.Length-1] -eq "\\") {
$oResource.url = $UrlPath.Replace("\\", "/") + "/" + $_.Name
} else {
$oResource.url = $UrlPath.Replace("\\", "/") + "/" + $_.Name
}
}
$lresource += $oResource
} else {
$Global:MartiErrorId = "MRI2001"
$message = "Document '$SourcePath' not found or is a folder"
Write-Log ($message + " " + $Global:MartiErrorId)
Close-Log
throw $message
}
Write-Log "Captured $($lresource.Count) items"
$oMarti.resources = $lresource
Close-Log
return $oMarti
}

View File

@ -0,0 +1,58 @@
function New-MartiChildItem
{
Param(
[Parameter(Mandatory)][String] $SourceFolder,
[String] $Filter ="*",
[String] $UrlPath,
[switch] $Recurse,
[switch] $ExtendAttributes,
[switch] $ExcludeHash,
[String] $LogPath
)
$script:LogPathName = $LogPath
Write-Debug "Parameter: LogPath Value: $LogPath "
Open-Log
Write-Log "Function 'New-MartiDefinition' parameters follow"
Write-Log "Parameter: SourceFolder Value: $SourceFolder "
Write-Log "Parameter: Filter Value: $Filter "
Write-Log "Parameter: Recurse Value: $Recurse "
Write-Log "Parameter: ExtendAttributes Value: $ExtendAttributes "
Write-Log "Parameter: ExcludeHash Value: $ExcludeHash "
Write-Log ""
$oMarti = New-MartiDefinition
$lresource = $oMarti.resources
$SourceFullName = (Get-Item -Path $SourceFolder).FullName
Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object {
$oResource = New-MartiItem -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash
if ($null -ne $UrlPath -and $UrlPath -ne "") {
$postfixName = $_.FullName.Replace($SourceFullName, "")
if ($postfixName[0] -eq "/" -or $postfixName[0] -eq "`\" ){
$postfixName = $postfixName.Substring(1, ($postfixName.Length-1))
}
if ($UrlPath[$UrlPath.Length-1] -eq "/" -or $UrlPath[$UrlPath.Length-1] -eq "`\") {
$oResource.url = $UrlPath.Replace("`\", "/") + $postfixName.Replace("`\", "/")
} else {
$oResource.url = $UrlPath.Replace("`\", "/") + "/" + $postfixName.Replace("`\", "/")
}
}
$lresource += $oResource
}
Write-Log "Captured $($lresource.Count) items"
$oMarti.resources = $lresource
Close-Log
return $oMarti
}

View File

@ -1,5 +1,5 @@
function Add-MartiItem
function New-MartiItem
{
Param(
[Parameter(Mandatory)][String] $SourcePath,
@ -38,7 +38,7 @@ Param(
$hash = (Get-FileHash -Path $item.FullName -Algorithm $hashAlgo).Hash
}
$lattribute = Get-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
$lattribute = Set-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
$oResource = [PSCustomObject]@{
title = $item.Name.Replace($item.Extension, "")