BSB Sample code

docs/samples/powershell/Invoke-BSBSample.ps1

@@ -1,6 +1,7 @@
 
 . C:\Users\meerkat\source\marti\source\powershell\New-Marti.ps1
-. C:\Users\meerkat\source\marti\source\powershell\Add-MartiItem.ps1
+. C:\Users\meerkat\source\marti\source\powershell\New-MartiChildItem.ps1
+. C:\Users\meerkat\source\marti\source\powershell\New-MartiItem.ps1
 . C:\Users\meerkat\source\marti\source\powershell\Get-Marti.ps1
 . C:\Users\meerkat\source\marti\source\powershell\Compress-Marti.ps1
 . C:\Users\meerkat\source\marti\source\powershell\Get-MartiFileAttributes.ps1
@@ -85,13 +86,30 @@ Write-Host "First fetch the BSB files " -ForeGroundColor Green
 $fileList = ListFtpDirectory -Username "anonymous" -Password "anon@merebox.com" -RemoteFile $remoteDirectory
 Write-Host "File list size: $($fileList.count)"
 
+
+Write-Host "Now iterate through the remote files and build remote marti list " -ForeGroundColor Green
+
 $oMarti = New-MartiDefinition
+$oMarti.title = "Remote_BSB_data"
+$oMarti.description = "This definition covers the remote BSB data files `r downloaded from the Australian Payment Network"
+$oMarti.contactPoint = "meerkat@merebox.com"
+$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json"
+$oMarti.theme = "payment"
+
 ForEach ($item in $fileList) {
     if ($item -ne "" -and $item.startswith("BSBDirectory")) {
-        #Write-Host "Pulling file: $item"
         PullFtpFile -Username "anonymous" -Password "anon@merebox.com" -RemoteFile ($remoteDirectory + $item) -OutputPath (Join-Path -Path $localDirectory -ChildPath $item)
         Write-Host "Add BSB $item file to Remote marti metadata sample " -ForeGroundColor Yellow
-        $oResource = Add-MartiItem -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes
+        $oResource = New-MartiItem -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes
+        if ($item.endswith(".txt")) {
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8
+        }
+        if ($item.endswith(".csv")) {
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0
+        }
         $oMarti.resources += $oResource
     }
 }
@@ -100,21 +118,51 @@ $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBRemote.mri.json"
 $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
 Write-Host "Remote marti definition file is $fileJson " -ForeGroundColor Green
 
-Write-Host "Now iterate through the local files and build ZIP " -ForeGroundColor Green
 
-if ($fileList -lt 0) {
+Write-Host "Now iterate through the local files and build marti ZIP " -ForeGroundColor Green
-    $zipFile = Join-Path -Path $localDirectory -ChildPath "BSBDirectory.zip"
+
-    if (Test-Path -Path $zipFile) {
+$oMarti = New-MartiDefinition
-        Remove-Item -Path $zipFile
+$oMarti.title = "Zip_BSB_data"
-    }
+$oMarti.description = "This definition covers the ZIP BSB data files `r downloaded from the Australian Payment Network"
-    foreach($file in Get-ChildItem $localDirectory)
+$oMarti.contactPoint = "meerkat@merebox.com"
-    {
+$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json"
-        if ($file.Name.startswith("BSBDirectory")) {
+$oMarti.theme = "payment"
-            Write-Host "Add BSB file $file to Local marti metadata sample " -ForeGroundColor Yellow
+
-            Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update
+$zipFileName = "BSBDirectory.zip"
+$zipFile = Join-Path -Path $localDirectory -ChildPath $zipFileName
+if (Test-Path -Path $zipFile) {
+    Remove-Item -Path $zipFile
+}
+foreach($file in Get-ChildItem $localDirectory)
+{
+    if ($file.Name.startswith("BSBDirectory") -and $file.Name -ne $zipFileName) {
+        Write-Host "Add BSB file $file to Local marti metadata sample " -ForeGroundColor Yellow
+        Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update
+        $oResource = New-MartiItem -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
+        if ($file.Extension -eq ".txt") {
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8
         }
+        if ($file.Extension -eq ".csv") {
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0
+        }
+        $oResource.url = "@"+$zipFileName + "/" + $file.Name
+        $oMarti.resources += $oResource
     }
 }
+$oResource = New-MartiItem -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
+Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "WINZIP"
+$oMarti.resources += $oResource
+
+$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mri.json"
+$oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
+Write-Host "ZIP marti definition file is $fileJson " -ForeGroundColor Green
+
+
+
+Write-Host "Now iterate through the local files with ZIP " -ForeGroundColor Green
 
 $oMarti = New-MartiChildItem -SourceFolder $localDirectory -UrlPath "./test" -Filter "BSBDirectory*" -LogPath ".\test\Logs" -ExtendAttributes
 $oMarti.title = "Local_BSB_data"

source/powershell/Get-Marti.ps1

@@ -64,13 +64,13 @@ function Get-SoftwareName {
 
 
 
-function Get-MartiItem
+function Get-MartChildResource
 {
     Param( 
-        [Parameter(Mandatory)][PSCustomObject] $MartiDefintiion,
+        [Parameter(Mandatory)][PSCustomObject] $Marti,
         [Parameter(Mandatory)][String] $Title,
-        [String] $DocumentName,
+        [String] $ResourceName,
-        [String] $Format,
+        [String] $Format = "*",
         [String] $LogPath
     
     ) 
@@ -80,12 +80,12 @@ function Get-MartiItem
     Write-Debug "Parameter: LogPath   Value: $LogPath "
     Open-Log
     Write-Log "Function 'Get-MartiItem' parameters follow"
-    Write-Log "Parameter: DocumentName   Value: $DocumentName "
+    Write-Log "Parameter: ResourceName   Value: $ResourceName "
-    Write-Log "Parameter: Filter   Value: $Filter "
+    Write-Log "Parameter: Format   Value: $Format "
     Write-Log ""
 
 
-    if ($null -eq $MartiDefintiion) {
+    if ($null -eq $Marti) {
         $Global:MartiErrorId = "MRI2101"
         $message = "No Marti definition supplied"
         Write-Log ($message + " " + $Global:MartiErrorId) 
@@ -94,7 +94,7 @@ function Get-MartiItem
     }
         
     
-    if ($null -eq $MartiDefintiion.resources -or $MartiDefintiion.resources.Count -lt 1) {
+    if ($null -eq $Marti.resources -or $Marti.resources.Count -lt 1) {
         $Global:MartiErrorId = "MRI2102"
         $message = "No documents listed"
         Write-Log ($message + " " + $Global:MartiErrorId) 
@@ -104,13 +104,13 @@ function Get-MartiItem
 
     [System.Collections.ArrayList]$lresource = @()
 
-    $MartiDefintiion.resources | ForEach-Object {
+    $Marti.resources | ForEach-Object {
 
-        if ($null -eq $Format -or $Format -eq "*" -or $Format -eq $_.format ) {
+        if ($Format -eq "*" -or $Format -eq $_.format ) {
             if ($Title -ne "*" -and $_.title -eq $Title) {
                 $lresource += $_
             } else {
-                if ($DocumentName -ne "*" -and $_.documentName -eq $DocumentName) {
+                if ($ResourceName -ne "*" -and $_.documentName -eq $ResourceName ) {
                     $lresource += $_
                 }
             }
@@ -122,3 +122,30 @@ function Get-MartiItem
     return $lresource
 }
 
+
+function Get-MartiResource {
+    Param (
+        # Marti definition
+        [Parameter(Mandatory)] [PSCustomObject] $Marti,
+        # Resource ID
+        [Parameter(Mandatory)] [String] $ResourceName
+    )
+
+    $script:LogPathName = $LogPath
+
+    Write-Debug "Parameter: LogPath   Value: $LogPath "
+    Open-Log
+    Write-Log "Function 'Get-MartiResource' parameters follow"
+    Write-Log "Parameter: ResourceName   Value: $ResourceName "
+    Write-Log ""
+
+    foreach ($item in $oMarti.resources) {
+        if ($item.uid -eq $ResourceName -or $item.documentName -eq $ResourceName){
+            Close-Log
+            return $item
+        }
+    }
+
+    Close-Log
+    return $null
+}

source/powershell/Get-MartiFileAttributes.ps1

@@ -145,10 +145,10 @@ function New-DefaultZipAttributes {
     return $lattribute
 }
 
-function Update-AttributeValueString {
+function Set-AttributeValueString {
     Param (
         # Attribute List
-        [Parameter(Mandatory)]  [System.Collections.ArrayList] $lAttribute,
+        [Parameter(Mandatory)]  [System.Collections.ArrayList] $Attributes,
         # Attribute Category
         [Parameter(Mandatory)] [String] $Category,
         # Attribute Key Name
@@ -161,7 +161,7 @@ function Update-AttributeValueString {
         [String] $Comparison = "EQ"
     )
 
-    foreach ($item in $lAttribute)
+    foreach ($item in $Attributes)
     {
         if ($item.category -eq $Category -and $item.name -eq $Key -and $item.function -eq $Function)
         {
@@ -173,11 +173,20 @@ function Update-AttributeValueString {
         }
     }
 
-    Write-Error "No match found for attribute name '$Key' and category '$Category' and function '$Function' "
+    # Add the attribute    
+    $oAttribute = [PSCustomObject]@{
+        category = $Category
+        name = $Key
+        function = $Function
+        comparison = $Comparison
+        value = $Value
+    }    
+    $Attributes += $oAttribute
+    return
 }
 
 
-function Update-AttributeValueNumber {
+function Set-AttributeValueNumber {
     Param (
         # Attribute List
         [Parameter(Mandatory)]  [System.Collections.ArrayList] $Attributes,
@@ -205,11 +214,21 @@ function Update-AttributeValueNumber {
         }
     }
 
-    Write-Error "No match found for attribute name '$Key' and category '$Category' and function '$Function' "
+
+    # Add the attribute    
+    $oAttribute = [PSCustomObject]@{
+        category = $Category
+        name = $Key
+        function = $Function
+        comparison = $Comparison
+        value = $Value
+    }    
+    $Attributes += $oAttribute
+    return
 }
 
 
-function Get-MartiFileAttributes {
+function Set-MartiFileAttributes {
     Param (
         # File path
         [Parameter(Mandatory)] [String] $Path,
@@ -235,8 +254,8 @@ function Get-MartiFileAttributes {
                 }
                 $rowCount += 1
             }
-            Update-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount
+            Set-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount
-            Update-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount
+            Set-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount
         }
     }
 
@@ -256,8 +275,8 @@ function Get-MartiFileAttributes {
                 }
                 $rowCount += 1
             }
-            Update-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount
+            Set-AttributeValueNumber -Attributes $lattribute -Key "records" -Category "dataset" -Function "count" -Value $rowCount
-            Update-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount
+            Set-AttributeValueNumber -Attributes $lattribute -Key "columns" -Category "dataset" -Function "count" -Value $colCount
         }
     }
 
@@ -271,7 +290,7 @@ function Get-MartiFileAttributes {
             $shell = New-Object -Com Shell.Application
             $zipFile = $shell.NameSpace($Path)
             $items = $zipFile.Items()
-            Update-AttributeValueNumber -Attributes $lattribute -Key "files" -Category "dataset" -Function "count" -Value $items.Count
+            Set-AttributeValueNumber -Attributes $lattribute -Key "files" -Category "dataset" -Function "count" -Value $items.Count
         }
     }
 
@@ -286,3 +305,4 @@ function Get-MartiFileAttributes {
     return $lattribute
 }
 
+

source/powershell/New-Marti.ps1

@@ -106,94 +106,6 @@ function New-MartiDefinition
 }
 
 
-
-
-function New-MartiChildItem
-{
-Param( 
-    [Parameter(Mandatory)][String] $SourceFolder,
-    [String] $Filter ="*",
-    [String] $UrlPath,
-    [switch] $Recurse,
-    [switch] $ExtendAttributes,
-    [switch] $ExcludeHash,
-    [String] $LogPath
-
-) 
-    $script:LogPathName = $LogPath
-
-    Write-Debug "Parameter: LogPath   Value: $LogPath "
-    Open-Log
-    Write-Log "Function 'New-MartiDefinition' parameters follow"
-    Write-Log "Parameter: SourceFolder   Value: $SourceFolder "
-    Write-Log "Parameter: Filter   Value: $Filter "
-    Write-Log "Parameter: Recurse   Value: $Recurse "
-    Write-Log "Parameter: ExcludeHash   Value: $ExcludeHash "
-    Write-Log ""
-
-    if ($ExcludeHash) {
-        $hashAlgo = ""
-    }
-    else {
-        $hashAlgo = "SHA256"
-    }
-    $version = "1.1.0"
-
-    $oMarti = New-MartiDefinition
-    $lresource = $oMarti.resources
-
-    $SourceFullName = (Get-Item -Path $SourceFolder).FullName
-
-    Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object {
-
-        Write-Log "Define file $($_.FullName) "
-        if ($ExcludeHash) {
-            $hash = ""
-        } else {
-            $hash = (Get-FileHash -Path $_.FullName -Algorithm $hashAlgo).Hash
-        }
-
-        $lattribute =  Get-MartiFileAttributes -Path $_.FullName -FileType $_.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
-
-        $oResource = [PSCustomObject]@{
-            title = $_.Name.Replace($_.Extension, "")
-            uid = (New-Guid).ToString()
-            documentName = $_.Name
-            issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
-            modified = $_.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")
-            state = "active"
-            author = ""
-            length = $_.Length
-            hash = $hash
-            hashAlgo = $hashAlgo   
-
-            description = ""
-            url = ""
-            version = $version
-            format = $_.Extension.Substring(1)
-            compression = ""
-            encryption = ""
-
-            attributes = $lattribute
-        }
-
-        if ($null -ne $UrlPath -and $UrlPath -ne "") {
-            $postfixName = $_.FullName.Replace($SourceFullName, "")
-            $oResource.url = Join-Path -Path $UrlPath -ChildPath $postfixName
-        }
-
-        $lresource += $oResource
-        
-    }
-    Write-Log "Captured $($lresource.Count) items"
-    $oMarti.resources = $lresource
-
-    Close-Log
-
-    return $oMarti
-
-}
-
 function Set-MartiAttribute
 {
 Param( 
@@ -233,95 +145,3 @@ Param(
     return $Attributes
 }
 
-function New-MartiItem
-{
-Param( 
-    [Parameter(Mandatory)][String] $SourcePath,
-    [String] $UrlPath = "",
-    [switch] $ExcludeHash,
-    [switch] $ExtendAttributes,
-    [String] $LogPath
-
-) 
-    $Global:MartiErrorId = ""
-    $script:LogPathName = $LogPath
-
-    Write-Debug "Parameter: LogPath   Value: $LogPath "
-    Open-Log
-    Write-Log "Function 'New-MartiItem' parameters follow"
-    Write-Log "Parameter: SourcePath   Value: $SourcePath "
-    Write-Log "Parameter: ExcludeHash   Value: $ExcludeHash "
-    Write-Log ""
-
-
-    if ($ExcludeHash) {
-        $hashAlgo = ""
-    }
-    else {
-        $hashAlgo = "SHA256"
-    }
-    $version = "1.1.0"
-
-    $oMarti = New-MartiDefinition
-    $lresource = $oMarti.resources
-
-    if (Test-Path -Path $SourcePath -PathType Leaf) {
-
-        $item = Get-Item -Path $SourcePath -Force 
-
-        Write-Log "Define file $($item.FullName) "
-        if ($ExcludeHash) {
-            $hash = ""
-        } else {
-            $hash = (Get-FileHash -Path $item.FullName -Algorithm $hashAlgo).Hash
-        }
-
-        $lattribute =  Get-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
-
-        $oResource = [PSCustomObject]@{ 
-            title = $item.Name.Replace($item.Extension, "")
-            uid = (New-Guid).ToString()
-            documentName = $item.Name
-            issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
-            modified = $item.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")
-            state = "active"
-            author = ""
-            length = $item.Length
-            hash = $hash
-            hashAlgo = $hashAlgo   
-
-            description = ""
-            url = ""
-            version = $version
-            format = $item.Extension.Substring(1)
-            compression = ""
-            encryption = ""
-
-            attributes = $lattribute
-        }
-
-        if ($null -ne $UrlPath -and $UrlPath -ne "") {
-            if ($UrlPath[$UrlPath.Length-1] -eq "/" -or $UrlPath[$UrlPath.Length-1] -eq "\\") {
-                $oResource.url = $UrlPath.Replace("\\", "/") + "/" + $_.Name
-            } else {
-                $oResource.url = $UrlPath.Replace("\\", "/") + "/" + $_.Name
-            }
-        }
-
-        $lresource += $oResource
-        
-    } else {
-        $Global:MartiErrorId = "MRI2001"
-        $message = "Document '$SourcePath' not found or is a folder"
-        Write-Log ($message + " " + $Global:MartiErrorId) 
-        Close-Log
-        throw $message
-    }
-    Write-Log "Captured $($lresource.Count) items"
-    $oMarti.resources = $lresource
-    Close-Log
-
-    return $oMarti
-
-}
-

source/powershell/New-MartiChildItem.ps1

@@ -0,0 +1,58 @@
+
+function New-MartiChildItem
+{
+Param( 
+    [Parameter(Mandatory)][String] $SourceFolder,
+    [String] $Filter ="*",
+    [String] $UrlPath,
+    [switch] $Recurse,
+    [switch] $ExtendAttributes,
+    [switch] $ExcludeHash,
+    [String] $LogPath
+
+) 
+    $script:LogPathName = $LogPath
+
+    Write-Debug "Parameter: LogPath   Value: $LogPath "
+    Open-Log
+    Write-Log "Function 'New-MartiDefinition' parameters follow"
+    Write-Log "Parameter: SourceFolder   Value: $SourceFolder "
+    Write-Log "Parameter: Filter   Value: $Filter "
+    Write-Log "Parameter: Recurse   Value: $Recurse "
+    Write-Log "Parameter: ExtendAttributes   Value: $ExtendAttributes "
+    Write-Log "Parameter: ExcludeHash   Value: $ExcludeHash "
+    Write-Log ""
+
+    $oMarti = New-MartiDefinition
+    $lresource = $oMarti.resources
+
+    $SourceFullName = (Get-Item -Path $SourceFolder).FullName
+
+    Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object {
+
+        $oResource = New-MartiItem -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash
+
+        if ($null -ne $UrlPath -and $UrlPath -ne "") {
+            $postfixName = $_.FullName.Replace($SourceFullName, "")
+            if ($postfixName[0] -eq "/" -or $postfixName[0] -eq "`\" ){
+                $postfixName = $postfixName.Substring(1, ($postfixName.Length-1))
+            }
+            if ($UrlPath[$UrlPath.Length-1] -eq "/" -or $UrlPath[$UrlPath.Length-1] -eq "`\") {
+                $oResource.url = $UrlPath.Replace("`\", "/") + $postfixName.Replace("`\", "/")
+            } else {
+                $oResource.url = $UrlPath.Replace("`\", "/") + "/" + $postfixName.Replace("`\", "/")
+            }
+        }
+
+        $lresource += $oResource
+        
+    }
+    Write-Log "Captured $($lresource.Count) items"
+    $oMarti.resources = $lresource
+
+    Close-Log
+
+    return $oMarti
+
+}
+

source/powershell/New-MartiItem.ps1

@@ -1,5 +1,5 @@
 
-function Add-MartiItem
+function New-MartiItem
 {
 Param( 
     [Parameter(Mandatory)][String] $SourcePath,
@@ -38,7 +38,7 @@ Param(
             $hash = (Get-FileHash -Path $item.FullName -Algorithm $hashAlgo).Hash
         }
 
-        $lattribute =  Get-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
+        $lattribute =  Set-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
 
         $oResource = [PSCustomObject]@{ 
             title = $item.Name.Replace($item.Extension, "")