Added Python code

.gitignore

@@ -12,3 +12,6 @@ buildNumber.properties
 
 test/powershell/results/
 docs/samples/powershell/test/
+
+docs/samples/python/test
+source/python/client/__pycache__

docs/samples/powershell/Invoke-SampleGenerateBsb.ps1

@@ -1,11 +1,10 @@
 
-. C:\Users\meerkat\source\marti\source\powershell\New-Marti.ps1
+. ..\..\..\source\powershell\New-Marti.ps1
-. C:\Users\meerkat\source\marti\source\powershell\New-MartiChildItem.ps1
+. ..\..\..\source\powershell\New-MartiChildItem.ps1
-. C:\Users\meerkat\source\marti\source\powershell\New-MartiItem.ps1
+. ..\..\..\source\powershell\New-MartiResource.ps1
-. C:\Users\meerkat\source\marti\source\powershell\Get-Marti.ps1
+. ..\..\..\source\powershell\Get-Marti.ps1
-. C:\Users\meerkat\source\marti\source\powershell\Compress-Marti.ps1
+. ..\..\..\source\powershell\Compress-Marti.ps1
-. C:\Users\meerkat\source\marti\source\powershell\Get-MartiFileAttributes.ps1
+. ..\..\..\source\powershell\Get-MartiResourceAttributes.ps1
-
 
 
 function PullFtpFile {
@@ -77,14 +76,17 @@ function ListFtpDirectory {
 }
 
 $remoteDirectory = "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/"
-# Change local directory
+
-$localDirectory = "./test/powershell/results/data"
+# Change local directory to suit
 $localDirectory = "./test"
+if (!(Test-Path -Path $localDirectory)) {
+    New-Item -Path $localDirectory
+}
 
 Write-Host "First fetch the BSB files " -ForeGroundColor Green
 
 $fileList = ListFtpDirectory -Username "anonymous" -Password "anon@merebox.com" -RemoteFile $remoteDirectory
-Write-Host "File list size: $($fileList.count)"
+Write-Host "File list size: $($fileList.count)" -ForegroundColor Gray
 
 
 Write-Host "Now iterate through the remote files and build remote marti list " -ForeGroundColor Green
@@ -93,14 +95,14 @@ $oMarti = New-MartiDefinition
 $oMarti.title = "Remote_BSB_data"
 $oMarti.description = "This definition covers the remote BSB data files `r downloaded from the Australian Payment Network"
 $oMarti.contactPoint = "meerkat@merebox.com"
-$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json"
+$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
 $oMarti.theme = "payment"
 
 ForEach ($item in $fileList) {
     if ($item -ne "" -and $item.startswith("BSBDirectory")) {
         PullFtpFile -Username "anonymous" -Password "anon@merebox.com" -RemoteFile ($remoteDirectory + $item) -OutputPath (Join-Path -Path $localDirectory -ChildPath $item)
         Write-Host "Add BSB $item file to Remote marti metadata sample " -ForeGroundColor Yellow
-        $oResource = New-MartiItem -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes
+        $oResource = New-MartiResource -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes
         if ($item.endswith(".txt")) {
             Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
             Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
@@ -117,6 +119,7 @@ ForEach ($item in $fileList) {
 $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBRemote.mri.json"
 $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
 Write-Host "Remote marti definition file is $fileJson " -ForeGroundColor Green
+Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray
 
 
 Write-Host "Now iterate through the local files and build marti ZIP " -ForeGroundColor Green
@@ -125,7 +128,7 @@ $oMarti = New-MartiDefinition
 $oMarti.title = "Zip_BSB_data"
 $oMarti.description = "This definition covers the ZIP BSB data files `r downloaded from the Australian Payment Network"
 $oMarti.contactPoint = "meerkat@merebox.com"
-$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json"
+$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
 $oMarti.theme = "payment"
 
 $zipFileName = "BSBDirectory.zip"
@@ -135,10 +138,10 @@ if (Test-Path -Path $zipFile) {
 }
 foreach($file in Get-ChildItem $localDirectory)
 {
-    if ($file.Name.startswith("BSBDirectory") -and $file.Name -ne $zipFileName) {
+    if ($file.Name.startswith("BSBDirectory") -and !($file.Name.EndsWith(".zip")) -and !($file.Name.EndsWith(".7z")) ) {
-        Write-Host "Add BSB file $file to Local marti metadata sample " -ForeGroundColor Yellow
+        Write-Host "Add BSB file $file to ZIP marti metadata sample " -ForeGroundColor Yellow
         Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update
-        $oResource = New-MartiItem -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
+        $oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
         if ($file.Extension -eq ".txt") {
             Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
             Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
@@ -152,13 +155,14 @@ foreach($file in Get-ChildItem $localDirectory)
         $oMarti.resources += $oResource
     }
 }
-$oResource = New-MartiItem -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
+$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
 Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "WINZIP"
 $oMarti.resources += $oResource
 
 $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mri.json"
 $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
 Write-Host "ZIP marti definition file is $fileJson " -ForeGroundColor Green
+Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray
 
 
 
@@ -168,11 +172,71 @@ $oMarti = New-MartiChildItem -SourceFolder $localDirectory -UrlPath "./test" -Fi
 $oMarti.title = "Local_BSB_data"
 $oMarti.description = "This definition covers the local BSB data files `r downloaded from the Australian Payment Network"
 $oMarti.contactPoint = "meerkat@merebox.com"
-$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json"
+$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
 $oMarti.theme = "payment"
 
 $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBLocal.mri.json"
 $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
 Write-Host "Local marti definition file is $fileJson " -ForeGroundColor Green
+Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray
+
+
+Write-Host "Now create an encrypted 7ZIP file with asymmetric password protection" -ForeGroundColor Green
+
+$oMarti = New-MartiDefinition
+$oMarti.title = "7ZIP_BSB_data"
+$oMarti.description = "This definition covers the 7ZIP BSB data files `r downloaded from the Australian Payment Network"
+$oMarti.contactPoint = "meerkat@merebox.com"
+$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
+$oMarti.theme = "payment"
+
+$zipFileName = "BSBDirectorySecure.7z"
+$zipFile = Join-Path -Path $localDirectory -ChildPath $zipFileName
+if (Test-Path -Path $zipFile) {
+    Remove-Item -Path $zipFile
+}
+foreach($file in Get-ChildItem $localDirectory)
+{
+    if ($file.Name.startswith("BSBDirectory") -and !($file.Name.EndsWith(".zip")) -and !($file.Name.EndsWith(".7z")) ) {
+        Write-Host "Add BSB file $file to 7ZIP marti metadata sample " -ForeGroundColor Yellow
+        if (Test-Path -Path $zipFile) {
+            Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip -Append 
+        } else {
+            Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip 
+        }
+        $oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
+        if ($file.Extension -eq ".txt") {
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8
+        }
+        if ($file.Extension -eq ".csv") {
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0
+        }
+        $oResource.url = "@"+$zipFileName + "/" + $file.Name
+        $oMarti.resources += $oResource
+    }
+}
+
+$noticeFile = Join-Path -Path  $localDirectory -ChildPath "README.txt"
+Set-Content -Path $noticeFile -Value "Generated by marti Samples"
+$oResource = New-MartiResource -SourcePath $noticeFile -UrlPath $localDirectory -LogPath ".\test\Logs"
+$oMarti.resources += $oResource
+
+$secret = "change_me_to_secure"
+Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $secret -EncryptFilenames
+
+$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
+$oResource.compression = "7ZIP"
+$oResource.encryption = New-Encryption -Algorithm "Passphrase" -Value $secret
+Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP"
+$oMarti.resources += $oResource
+
+$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBSecure.mri.json"
+$oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
+Write-Host "Secure 7ZIP marti definition file is $fileJson " -ForeGroundColor Green
+Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray
+
 
 Write-Host "Sample execution completed" -ForeGroundColor Green

docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1

@@ -0,0 +1,175 @@
+
+. ..\..\..\source\powershell\New-Marti.ps1
+. ..\..\..\source\powershell\New-MartiChildItem.ps1
+. ..\..\..\source\powershell\New-MartiResource.ps1
+. ..\..\..\source\powershell\Get-Marti.ps1
+. ..\..\..\source\powershell\Compress-Marti.ps1
+. ..\..\..\source\powershell\Get-MartiResourceAttributes.ps1
+
+
+function PullFtpFile {
+    param (
+        [String] $RemoteFile,
+        [String] $OutputPath,
+        [String] $Username,
+        [String] $Password,
+        [int] $Buffersize = 1024
+    )
+    
+
+
+    $FTPRequest = [System.Net.FtpWebRequest]::Create($RemoteFile)
+    $FTPRequest.Credentials = New-Object System.Net.NetworkCredential($Username,$Password)
+    $FTPRequest.Method = [System.Net.WebRequestMethods+Ftp]::DownloadFile
+    $FTPRequest.UseBinary = $true
+    $FTPRequest.KeepAlive = $false
+
+    $FTPResponse = $FTPRequest.GetResponse()
+    $ResponseStream = $FTPResponse.GetResponseStream()
+
+    $LocalFileStream = New-Object IO.FileStream ($OutputPath,[IO.FileMode]::Create)
+    if ($null -eq $LocalFileStream) {
+        Write-Host "Write failed to file $OutputPath"
+        return
+    }
+    [byte[]]$ReadBuffer = New-Object byte[] $Buffersize
+
+    # Loop through the download
+    do {
+        $ReadLength = $ResponseStream.Read($ReadBuffer,0,$Buffersize)
+        $LocalFileStream.Write($ReadBuffer,0,$ReadLength)
+    }
+    while ($ReadLength -gt 0)
+    $LocalFileStream.close()
+}
+
+
+function ListFtpDirectory {
+    param (
+        [String] $RemoteFile,
+        [String] $Username,
+        [String] $Password,
+        [int] $Buffersize = 1024
+    )
+    
+    $FTPRequest = [System.Net.FtpWebRequest]::Create($RemoteFile)
+    $FTPRequest.Credentials = New-Object System.Net.NetworkCredential($Username,$Password)
+    $FTPRequest.Method = [System.Net.WebRequestMethods+Ftp]::ListDirectory
+    $FTPRequest.UseBinary = $false
+    $FTPRequest.KeepAlive = $false
+
+    $FTPResponse = $FTPRequest.GetResponse()
+    $ResponseStream = $FTPResponse.GetResponseStream()
+
+    $Encoding = new-object System.Text.AsciiEncoding
+    $ListBuffer = ""
+    [byte[]]$ReadBuffer = New-Object byte[] $Buffersize
+
+    do {
+        $ReadLength = $ResponseStream.Read($ReadBuffer,0,$Buffersize)
+        $ListBuffer += ($Encoding.GetString($ReadBuffer, 0, $ReadLength))
+    }
+    while ($ReadLength -ne 0)
+
+    $list = $ListBuffer.Split([Environment]::NewLine)
+    return $list
+}
+
+function New-RandomPassword {
+    param(
+        [int] $length = 30,
+        [String] $characters = "abcdefghiklmnoprstuvwxyzABCDEFGHKLMNOPRSTUVWXYZ1234567890!()?}][{@#*+-",
+        [switch] $ConvertToSecureString
+    )
+    $password = 1..$length | ForEach-Object { Get-Random -Maximum $characters.length }
+
+    $private:ofs=""
+    if ($ConvertToSecureString.IsPresent) {
+        return ConvertTo-SecureString -String [String]$characters[$password] -AsPlainText -Force
+    } else {
+        return [String]$characters[$password]
+    }
+}
+    
+
+
+
+Write-Host "Please execute in the same directory as script" -ForeGroundColor Yellow
+
+$recipientKey = "CN=PeterDocs"
+$remoteDirectory = "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/"
+
+# Change local directory to suit
+$localDirectory = "./test"
+if (!(Test-Path -Path $localDirectory)) {
+    New-Item -Path $localDirectory
+}
+
+Write-Host "First fetch the BSB files " -ForeGroundColor Green
+
+$fileList = ListFtpDirectory -Username "anonymous" -Password "anon@merebox.com" -RemoteFile $remoteDirectory
+Write-Host "File list size: $($fileList.count)" -ForegroundColor Gray
+
+Write-Host "Now create an encrypted 7ZIP file with password " -ForeGroundColor Green
+
+$oMarti = New-MartiDefinition
+$oMarti.title = "7ZIP_BSB_data"
+$oMarti.description = "This definition covers the 7ZIP BSB data files `r downloaded from the Australian Payment Network"
+$oMarti.contactPoint = "meerkat@merebox.com"
+$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
+$oMarti.theme = "payment"
+
+$zipFileName = "BSBDirectoryPKI.7z"
+$zipFile = Join-Path -Path $localDirectory -ChildPath $zipFileName
+if (Test-Path -Path $zipFile) {
+    Remove-Item -Path $zipFile
+}
+foreach($file in Get-ChildItem $localDirectory)
+{
+    if ($file.Name.startswith("BSBDirectory") -and $file.Name.EndsWith(".csv") ) {
+        Write-Host "Add BSB file $file to 7ZIP marti metadata sample " -ForeGroundColor Yellow
+        if (Test-Path -Path $zipFile) {
+            Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip -Append 
+        } else {
+            Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip 
+        }
+        $oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
+        if ($file.Extension -eq ".txt") {
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8
+        }
+        if ($file.Extension -eq ".csv") {
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0
+            Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0
+        }
+        $oResource.url = "@"+$zipFileName + "/" + $file.Name
+        $oMarti.resources += $oResource
+    }
+}
+
+$noticeFile = Join-Path -Path  $localDirectory -ChildPath "README.txt"
+Set-Content -Path $noticeFile -Value "Generated by marti Samples"
+$oResource = New-MartiResource -SourcePath $noticeFile -UrlPath $localDirectory -LogPath ".\test\Logs"
+$oMarti.resources += $oResource
+
+$secret = New-RandomPassword -Length 80
+$encryptedSecret = Protect-CmsMessage -To $recipientKey -Content $secret 
+
+Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $secret -EncryptFilenames
+
+$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
+$oResource.compression = "7ZIP"
+$oResource.encryption = New-Encryption -Algorithm "PKI" -Value $($encryptedSecret)
+
+Write-Debug "Secret: $secret"
+
+Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP"
+$oMarti.resources += $oResource
+
+$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSB7ZipPKI.mri.json"
+$oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson
+Write-Host "7ZIP marti definition file is $fileJson " -ForeGroundColor Green
+Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray
+
+Write-Host "Sample secure execution completed" -ForeGroundColor Green

docs/samples/python/SampleGenerateBsb.py

@@ -0,0 +1,75 @@
+
+
+import ftplib 
+import os
+import json
+import sys
+  
+sys.path.insert(0, "../../../source/python/client")
+from marti import *
+
+def ftpList(host, path):
+    
+    files = []
+    with ftplib.FTP(host) as ftp:
+        try:
+            ftp.login()  
+            ftp.cwd(path)
+            files = ftp.nlst()
+        except ftplib.all_errors as e:
+            print('FTP error:', e)
+
+    return files
+
+
+def ftpPull(host, file_remote, file_local):
+
+    with ftplib.FTP(host) as ftp:
+                
+        try:
+            ftp.login()  
+            ftp.sendcmd('TYPE I') 
+            
+            with open(file_local, 'w') as fl:
+                res = ftp.retrlines('RETR ' + file_remote, fl.write)
+                if not res.startswith('226 Transfer complete'):
+                    print('Download failed')
+                    if os.path.isfile(file_local):
+                        os.remove(file_local)          
+
+        except ftplib.all_errors as e:
+            print('FTP error:', e) 
+            if os.path.isfile(file_local):
+                os.remove(file_local)
+
+
+remote_host = 'bsb.hostedftp.com'
+remote_dir = '/~auspaynetftp/BSB/'
+
+print("Fetch sample file list")
+files = ftpList(remote_host, remote_dir)
+
+print("Fetch sample files")
+for file_name in files:
+    if file_name.startswith('BSBDirectory'):
+        if file_name.endswith('.csv') | file_name.endswith('.txt'):
+            file_remote = remote_dir + file_name
+            file_local = './test/' + file_name
+            ftpPull(remote_host, file_remote, file_local)
+
+print("Creating marti definition")
+oMarti = NewMartiDefinition()
+
+for file_name in files:
+    if file_name.startswith('BSBDirectory'):
+        if file_name.endswith('.csv') | file_name.endswith('.txt'):
+            oResource = NewMartiResource(os.path.join("./test/", file_name), "", False, True, "./test/logs")
+            oMarti["resources"].append(oResource)
+
+print("Save marti definition")
+jd = json.dumps(oMarti, indent=5)
+
+jsonFile = open("./test/BSBDirectoryPlain.mri.json", "w")
+jsonFile.write(jd)
+jsonFile.close()
+print("Sample completed")

source/powershell/Get-MartiResourceAttributes.ps1

@@ -228,7 +228,7 @@ function Set-AttributeValueNumber {
 }
 
 
-function Set-MartiFileAttributes {
+function Set-MartiResourceAttributes {
     Param (
         # File path
         [Parameter(Mandatory)] [String] $Path,

source/powershell/New-MartiChildItem.ps1

@@ -30,7 +30,7 @@ Param(
 
     Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object {
 
-        $oResource = New-MartiItem -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash
+        $oResource = New-MartiResource -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash
 
         if ($null -ne $UrlPath -and $UrlPath -ne "") {
             $postfixName = $_.FullName.Replace($SourceFullName, "")

source/powershell/New-MartiItem.ps1

@@ -1,6 +1,5 @@
 
-function New-MartiItem
+function New-MartiResource {
-{
 Param( 
     [Parameter(Mandatory)][String] $SourcePath,
     [String] $UrlPath = "",
@@ -9,12 +8,14 @@ Param(
     [String] $LogPath
 
 ) 
+
     $Global:MartiErrorId = ""
     $script:LogPathName = $LogPath
  
     Write-Debug "Parameter: LogPath   Value: $LogPath "
     Open-Log
-    Write-Log "Function 'Add-MartiItem' parameters follow"
+    Write-Log "Function 'New-MartiResource' parameters follow"
+    Write-Log "Parameter: UrlPath   Value: $UrlPath "
     Write-Log "Parameter: SourcePath   Value: $SourcePath "
     Write-Log "Parameter: ExcludeHash   Value: $ExcludeHash "
     Write-Log ""
@@ -27,18 +28,12 @@ Param(
         Write-Log "Define file $($item.FullName) "
 
         if ($ExcludeHash) {
-            $hashAlgo = ""
+            $hash = $null
-        }
-        else {
-            $hashAlgo = "SHA256"
-        }
-        if ($ExcludeHash) {
-            $hash = ""
         } else {
-            $hash = (Get-FileHash -Path $item.FullName -Algorithm $hashAlgo).Hash
+            $hash = New-MartiHash -Algorithm "SHA256" -FilePath $item.FullName
         }
 
-        $lattribute =  Set-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
+        $lattribute =  Set-MartiResourceAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
 
         $oResource = [PSCustomObject]@{ 
             title = $item.Name.Replace($item.Extension, "")
@@ -50,14 +45,13 @@ Param(
             author = ""
             length = $item.Length
             hash = $hash
-            hashAlgo = $hashAlgo   
 
             description = ""
             url = ""
             version = $version
             format = $item.Extension.Substring(1)
-            compression = ""
+            compression = $null
-            encryption = ""
+            encryption = $null
 
             attributes = $lattribute
         }
@@ -83,3 +77,36 @@ Param(
 
 }
 
+function New-MartiHash{
+    Param( 
+        [Parameter(Mandatory)][String] $Algorithm,
+        [String] $FilePath
+        [String] $Value = ""
+    ) 
+
+    if ($Value  -eq "" -and $FilePath -ne "") {
+        $Value = (Get-FileHash -Path $FilePath -Algorithm $Algorithm).Hash
+    }
+
+    $oHash = [PSCustomObject]@{ 
+        algo = $Algorithm
+        value = $Value
+    }
+
+    return $oHash
+}
+    
+function New-Encryption{
+Param( 
+    [Parameter(Mandatory)][String] $Algorithm,
+    [String] $Value
+
+) 
+    
+    $oEncryption = [PSCustomObject]@{ 
+        algo = $Algorithm
+        value = $Value
+    }
+
+    return $oEncryption
+}

source/python/client/marti.py

@@ -0,0 +1,525 @@
+
+
+import ftplib
+from genericpath import getsize 
+import os
+import uuid
+import json
+import datetime
+import getpass
+import hashlib
+
+global_LogPathName = ""
+global_SoftwareVersion = "0.0.1"
+global_default_metaFile = "##marti##.mri"
+
+global_MartiErrorId = ""
+
+
+def GetSoftwareName():
+    return "MARTIREFERENCE"
+
+
+def GetLogName():
+
+    today = datetime.datetime.today() 
+    dateToday = today.strftime("%Y-%m-%d") 
+    
+    if None ==  global_LogPathName or global_LogPathName == "":
+        return None
+
+    if not os.path.exists(global_LogPathName):
+        os.mkdir(global_LogPathName)
+    
+    logName = GetSoftwareName() + "_" + dateToday + ".log"
+
+    return os.path.join(global_LogPathName, logName)
+
+
+def WriteLog(LogEntry):
+
+    sFullPath = GetLogName()
+
+    today = datetime.datetime.today() 
+    dateToday = today.strftime("%Y-%m-%d") 
+
+    if None != sFullPath and sFullPath != "":
+
+        if not os.path.exists(sFullPath):
+            print("Log path: $sFullPath")
+            filec = open(sFullPath, "a")
+        else:
+            filec = open(sFullPath, "a")
+        
+        filec.write(dateToday)
+        filec.write(".")
+        filec.write(LogEntry)
+        filec.write("\n")
+        filec.close()
+    
+
+def OpenLog():
+
+    today = datetime.datetime.today() 
+    dateToday = today.strftime("%Y-%m-%d") 
+    WriteLog("***********************************************************************************")
+    WriteLog("*   Start of processing: {}".format(dateToday))
+    WriteLog("***********************************************************************************")
+
+
+def CloseLog():
+
+    today = datetime.datetime.today() 
+    dateToday = today.strftime("%Y-%m-%d") 
+    WriteLog("***********************************************************************************")
+    WriteLog("*   End of processing: {}".format(dateToday))
+    WriteLog("***********************************************************************************")
+
+
+def NewMartiDefinition():
+
+    today = datetime.datetime.today() 
+    dateToday = today.strftime("%Y-%m-%d") 
+
+    oSoftware = {
+        "extension": "software",
+        "softwareName": GetSoftwareName(),
+        "author": "Meerkat@merebox.com",
+        "version": global_SoftwareVersion
+    }
+
+    publisher = getpass.getuser()
+
+    lcustom = []
+    lcustom.append(oSoftware)
+
+    lresource = []
+
+    oMarti = {
+        "title": "",
+        "uid": str(uuid.uuid4()),
+        "resources": lresource,
+
+        "description": "",
+        "modified": dateToday,
+        "tags": ["document", "marti"],
+        "publisher": publisher,
+        "contactPoint": "",
+        "accessLevel": "Confidential",
+        "rights": "Restricted",
+        "license": "",
+        "state": "active",
+
+        "describedBy": "",
+        "landingPage": "",
+        "theme": "", 
+
+        "custom": lcustom
+    }
+
+    return oMarti
+
+
+
+def NewMartiResource(SourcePath, UrlPath, ExcludeHash, ExtendAttributes, LogPath): 
+
+    today = datetime.datetime.today() 
+    dateToday = today.strftime("%Y-%m-%d") 
+
+    global global_MartiErrorId
+    global_MartiErrorId = ""
+    global global_LogPathName
+    global_LogPathName = LogPath
+
+    OpenLog()
+    WriteLog("Function 'NewMartiResource' parameters follow")
+    WriteLog("Parameter: SourcePath   Value: {}".format(SourcePath))
+    WriteLog("Parameter: UrlPath   Value: {}".format(UrlPath))
+    WriteLog("Parameter: ExcludeHash   Value: {}".format(ExcludeHash))
+    WriteLog("")
+
+    if os.path.exists(SourcePath):
+       
+        item = os.path.basename(SourcePath)
+
+        WriteLog("Define file {}".format(SourcePath))
+
+        try:
+            mtime = os.path.getmtime(SourcePath)
+        except OSError:
+            mtime = 0
+        last_modified_date = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S")
+
+        if ExcludeHash:
+            hash = None
+        else:
+            hash = NewMartiHash(Algorithm="SHA256", FilePath=SourcePath, Value="")
+
+        lattribute = SetMartiResourceAttributes(SourcePath, os.path.splitext(SourcePath)[1][1:], ExtendAttributes)
+
+        oResource = { 
+            "title": item.replace(os.path.splitext(SourcePath)[1], ""),
+            "uid": str(uuid.uuid4()), 
+            "documentName": item,
+            "issuedDate": dateToday,
+            "modified": last_modified_date,
+            "state": "active",
+            "author": "",
+            "length": os.path.getsize(SourcePath),
+            "hash": hash,
+
+            "description": "",
+            "url": "",
+            "version": global_SoftwareVersion,
+            "format": os.path.splitext(SourcePath)[1][1:],
+            "compression": None,
+            "encryption": None,
+
+            "attributes": lattribute
+        }
+
+        if None != UrlPath and UrlPath != "":
+            if UrlPath[UrlPath.Length-1] == "/" or UrlPath[UrlPath.Length-1] == "\\":
+                oResource["url"] = UrlPath.replace("\\", "/") + item
+            else:
+                oResource["url"] = UrlPath.replace("\\", "/") + "/" + item
+        
+        
+    else:
+        global_MartiErrorId = "MRI2001"
+        message = "Document '{}' not found or is a folder".format(SourcePath)
+        WriteLog(message + " " + global_MartiErrorId) 
+        CloseLog()
+        raise Exception(message)
+    
+    CloseLog()
+
+    return oResource
+
+
+
+def NewMartiHash(Algorithm, FilePath, Value=""):
+    
+    if Value  == "" and FilePath != "":
+        if Algorithm == "SHA256":
+            sha_hash = hashlib.sha256()
+            with open(FilePath,"rb") as fh:
+                for byte_block in iter(lambda: fh.read(4096),b""):
+                    sha_hash.update(byte_block)
+                Value = sha_hash.hexdigest()
+        if Algorithm == "SHA512":
+            sha_hash = hashlib.sha512()
+            with open(FilePath,"rb") as fh:
+                for byte_block in iter(lambda: fh.read(4096),b""):
+                    sha_hash.update(byte_block)
+                Value = sha_hash.hexdigest()
+        if Algorithm == "MD5":
+            sha_hash = hashlib.md5()
+            with open(FilePath,"rb") as fh:
+                for byte_block in iter(lambda: fh.read(4096),b""):
+                    sha_hash.update(byte_block)
+                Value = sha_hash.hexdigest()
+
+    oHash = { 
+        "algo": Algorithm,
+        "value": Value
+    }
+
+    return oHash
+    
+
+def NewEncryption(Algorithm, Value):
+
+    oEncryption = { 
+        "algo": Algorithm,
+        "value": Value
+    }
+
+    return oEncryption
+
+
+
+
+
+
+def SetMartiAttribute(Attributes, ACategory, AName, AFunction, Comparison, Value):
+
+    matched = False
+   
+    for attr in Attributes:
+
+        if attr["category"] == ACategory and attr["name"] == AName and attr["function"] == AFunction:
+            matched = True
+            attr["comparison"] = Comparison
+            attr["value"] = Value
+        
+
+    if not matched:
+        
+        oAttribute = {
+            "category": ACategory,
+            "name": AName,
+            "function": AFunction,
+            "comparison": Comparison,
+            "value": Value
+        }
+
+        Attributes.append(oAttribute)
+    
+
+    return Attributes
+
+
+
+def NewDefaultCsvAttributes():
+       
+    lattribute = []
+
+    oAttribute = {
+        "category": "dataset",
+        "name": "header",
+        "function": "count",
+        "comparison": "NA",
+        "value": 1
+    }
+    lattribute.append(oAttribute)
+    
+    oAttribute = {
+        "category": "dataset",
+        "name": "footer",
+        "function": "count",
+        "comparison": "NA",
+        "value":  0
+    }
+    lattribute.append(oAttribute)
+    
+    oAttribute = {
+        "category": "format",
+        "name": "separator",
+        "function": "value",
+        "comparison": "NA",
+        "value": ","
+    }
+    lattribute.append(oAttribute)
+    
+    oAttribute = {
+        "category": "format",
+        "name": "columns",
+        "function": "value",
+        "comparison": "NA",
+        "value": ","
+    }
+    lattribute.append(oAttribute)
+
+    oAttribute = {
+        "category": "dataset",
+        "name": "records",
+        "function": "count",
+        "comparison": "NA",
+        "value": 0
+    }
+    lattribute.append(oAttribute)
+    
+    oAttribute = {
+        "category": "dataset",
+        "name": "columns",
+        "function": "count",
+        "comparison": "NA",
+        "value": 0
+    }
+    lattribute.append(oAttribute)
+    
+    return lattribute
+
+
+
+def NewDefaultJsonAttributes():
+       
+    lattribute = []
+    
+    oAttribute = {
+        "category": "format",
+        "name": "list",
+        "function": "offset",
+        "comparison": "NA",
+        "value": ","
+    }
+    lattribute.append(oAttribute)
+    
+    oAttribute = {
+        "category": "format",
+        "name": "columns",
+        "function": "value",
+        "comparison": "NA",
+        "value": ","
+    }
+    lattribute.append(oAttribute)
+
+    oAttribute = {
+        "category": "dataset",
+        "name": "records",
+        "function": "count",
+        "comparison": "NA",
+        "value": 0
+    }    
+    lattribute.append(oAttribute)
+    
+    oAttribute = {
+        "category": "dataset",
+        "name": "columns",
+        "function": "count",
+        "comparison": "NA",
+        "value": 0
+    }
+    lattribute.append(oAttribute)
+    
+    return lattribute
+
+
+def NewDefaultZipAttributes(CompressionType = "ZIP",Encryption = ""):
+       
+    lattribute = []
+    
+    oAttribute = {
+        "category": "format",
+        "name": "compression",
+        "function": "algorithm",
+        "comparison": "NA",
+        "value": CompressionType
+    }
+    lattribute.append(oAttribute)
+    
+    oAttribute = {
+        "category": "format",
+        "name": "encryption",
+        "function": "algorithm",
+        "comparison": "NA",
+        "value": Encryption
+    }
+    lattribute.append(oAttribute)
+
+    oAttribute = {
+        "category": "dataset",
+        "name": "files",
+        "function": "count",
+        "comparison": "NA",
+        "value": 0
+    }    
+    lattribute.append(oAttribute)
+        
+    return lattribute
+
+
+def SetAttributeValueString(Attributes,Category,Key,Function,Value,Comparison = "EQ"):
+
+    for item in Attributes:
+    
+        if item["category"] == Category and item["name"] == Key and item["function"] == Function:
+            if item["comparison"] == "NA" or item["comparison"] == Comparison:
+                item["comparison"] = Comparison
+                item["value"] = Value
+                return
+
+    # Add the attribute    
+    oAttribute = {
+        "category": Category,
+        "name": Key,
+        "function": Function,
+        "comparison": Comparison,
+        "value": Value
+    }    
+
+    Attributes.append(oAttribute)
+    
+    return
+
+
+def SetAttributeValueNumber(Attributes,Category,Key,Function,Value,Comparison = "EQ"):
+
+    for item in Attributes:
+    
+        if item["category"] == Category and item["name"] == Key and item["function"] == Function:
+            if item["comparison"] == "NA" or item["comparison"] == Comparison:
+                item["comparison"] = Comparison
+                item["value"] = Value
+                return
+
+
+    # Add the attribute    
+    oAttribute = {
+        "category": Category,
+        "name": Key,
+        "function": Function,
+        "comparison": Comparison,
+        "value": Value
+    }    
+
+    Attributes.append(oAttribute)
+
+    return
+
+
+def SetMartiResourceAttributes(Path,FileType,ExtendedAttributes):
+
+    lattribute = None
+
+    if FileType == "csv":
+        lattribute = NewDefaultCsvAttributes()
+
+        if ExtendedAttributes:
+            delimiter = ","
+            rowCount = 0
+            colCount = 0
+            # csvData = Import-Csv $Path -Delimiter $delimiter 
+            # foreach ($datum in $csvData) {
+            #     $cc = (Get-Member -InputObject $datum -type NoteProperty).count
+            #     if ($colCount -lt $cc) {
+            #         $colCount = $cc
+            #     }
+            #     $rowCount += 1
+            # }
+            SetAttributeValueNumber(lattribute, "records", "dataset", "count", rowCount)
+            SetAttributeValueNumber(lattribute, "columns", "dataset", "count", colCount)
+
+
+    if FileType == "txt":
+        lattribute = NewDefaultCsvAttributes()
+
+        if ExtendedAttributes:
+            delimiter = "`t"
+            rowCount = 0
+            colCount = 0
+            # $csvData = Import-Csv $Path -Delimiter $delimiter 
+            # foreach ($datum in $csvData) {
+            #     $cc = (Get-Member -InputObject $datum -type NoteProperty).count
+            #     if ($colCount -lt $cc) {
+            #         $colCount = $cc
+            #     }
+            #     $rowCount += 1
+            # }
+            SetAttributeValueNumber(lattribute, "records", "dataset", "count", rowCount)
+            SetAttributeValueNumber(lattribute, "columns", "dataset", "count", colCount)
+
+
+    if FileType == "json":
+        lattribute = NewDefaultJsonAttributes()
+    
+
+    if FileType == "zip":
+        lattribute = NewDefaultZipAttributes("ZIP")
+        if ExtendedAttributes:
+            # $shell = New-Object -Com Shell.Application
+            # $zipFile = $shell.NameSpace($Path)
+            # $items = $zipFile.Items()
+            SetAttributeValueNumber(lattribute, "files", "dataset", "count", -1)
+
+    if FileType == "7z":
+        lattribute = NewDefaultZipAttributes("7Z")
+
+    if lattribute == None:
+        lattribute = []
+    
+
+    return lattribute
+
+

test/powershell/test_Marti.ps1

@@ -20,7 +20,7 @@ Write-Host "Get item Title: $($y.title)"
 Write-Host "Get item Url: $($y.url)"
 
 Write-Host "Test case #4"
-$oMarti = New-MartiItem -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs"
+$oMarti = New-MartiResource -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs"
 $oMarti.description = "Sample execution for ckan"
 
 $x = ConvertTo-Json -InputObject $oMarti
@@ -36,7 +36,7 @@ $x = ConvertTo-Html -InputObject $oMarti
 Set-Content -Path ".\test\powershell\results\marti_test02.mri.html" -Value $x
 
 Write-Host "Test case #5"
-$oMarti = New-MartiItem -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs"
+$oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs"
 $oMarti.description = "Sample execution with error"
 
 $x = ConvertTo-Json -InputObject $oMarti

test/powershell/test_MartiData2.ps1

@@ -7,7 +7,7 @@
 $bsbFile = ".\test\powershell\results\data\bsb.csv"
 
 Write-Host ">>>>>>Test case #1"
-$x = New-MartiItem -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
+$x = New-MartiResource -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
 
 Write-Host ">>>>>>Test case #2"
 $x.resources
@@ -25,7 +25,7 @@ $y
 $covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
 
 Write-Host ">>>>>>Test case #5"
-$x = New-MartiItem -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
+$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
 
 Write-Host ">>>>>>Test case #6"
 $x.resources

test/powershell/test_MartiData3.ps1

@@ -6,7 +6,7 @@
 
 $covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
 
-$x = New-MartiItem -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
+$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
 
 [System.Collections.ArrayList] $attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 516
 $attr = Set-MartiAttribute -Attributes $attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 12